From 3db9d9ab336183e220da84f5b9bdfc3119f6933e Mon Sep 17 00:00:00 2001 From: Jason Pereira Date: Tue, 20 Sep 2022 15:09:36 +0100 Subject: [PATCH] Add support for Journald input (#302) * add support for journald * fix syntax error * fix syntax error 2 * fix incorrect ordering of if statements in template * appease the test suite * update readme --- README.md | 4 + manifests/input.pp | 18 ++-- manifests/params.pp | 11 +- templates/filestream.yml.erb | 203 ----------------------------------- templates/input.yml.erb | 57 +++++++++- 5 files changed, 70 insertions(+), 223 deletions(-) delete mode 100644 templates/filestream.yml.erb diff --git a/README.md b/README.md index e7c40eb..ce3d5a5 100644 --- a/README.md +++ b/README.md @@ -400,6 +400,10 @@ to fully understand what these parameters do. [See above](#multiline-logs). (default: {}) - `host`: [String] Host and port used to read events for TCP or UDP plugin (default: localhost:9000) - `max_message_size`: [String] The maximum size of the message received over TCP or UDP (default: undef) + - `keep_null`: [Boolean] If this option is set to true, fields with null values will be published in the output document (default: undef) + - `include_matches`: [Array] Journald input only, A collection of filter expressions used to match fields. The format of the expression is field=value (default: []) + - `seek`: [Enum] Journald input only, The position to start reading the journal from (default: undef) + - `index`: [String] If present, this formatted string overrides the index for events from this input (for elasticsearch outputs), or sets the raw_index field of the event’s metadata (for other outputs) (default: undef) ## Limitations This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping diff --git a/manifests/input.pp b/manifests/input.pp index a5e0b37..58a4528 100644 --- a/manifests/input.pp +++ b/manifests/input.pp @@ -51,19 +51,17 @@ Array $processors = [], Boolean $pure_array = false, String $host = 'localhost:9000', + Boolean $keep_null = false, + Array[String] $include_matches = [], + Optional[Enum['head', 'tail', 'cursor']] $seek = undef, Optional[String] $max_message_size = undef, + Optional[String] $index = undef, ) { - if Integer($filebeat::major_version) < 8 { - if versioncmp($facts['filebeat_version'], '7.16') > 0 { - $input_template = 'filestream.yml.erb' - } elsif versioncmp($facts['filebeat_version'], '6') > 0 { - $input_template = 'input.yml.erb' - } else { - $input_template = 'prospector.yml.erb' - } + if versioncmp($facts['filebeat_version'], '6') > 0 { + $input_template = 'input.yml.erb' } else { - $input_template = 'filestream.yml.erb' + $input_template = 'prospector.yml.erb' } if 'filebeat_version' in $facts and $facts['filebeat_version'] != false { @@ -119,7 +117,7 @@ $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { true => undef, default => $filebeat::major_version ? { - '5' => "/usr/local/sbin/filebeat -N -configtest -c %", + '5' => '/usr/local/sbin/filebeat -N -configtest -c %', default => "/usr/local/sbin/filebeat -c ${filebeat::config_file} test config", }, } diff --git a/manifests/params.pp b/manifests/params.pp index 468500a..8f8003d 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -169,13 +169,10 @@ } } - if Integer($major_version) < 8 { - if versioncmp($facts['filebeat_version'], '7.16') > 0 { - $default_input_type = 'filestream' - } else { - $default_input_type = 'log' - } - } else { + # filestream input type added in 7.10, deprecated in 7.16 + if versioncmp($facts['filebeat_version'], '7.10') > 0 { $default_input_type = 'filestream' + } else { + $default_input_type = 'log' } } diff --git a/templates/filestream.yml.erb b/templates/filestream.yml.erb deleted file mode 100644 index fa07267..0000000 --- a/templates/filestream.yml.erb +++ /dev/null @@ -1,203 +0,0 @@ -<%- if @pure_array -%> -<%= scope['filebeat::inputs'].to_yaml() %> -<%- else -%> ---- -- type: <%= @input_type %> - id: <%= @name %> - <%- if @input_type == 'syslog' -%> - protocol.<%= @syslog_protocol %>: - host: <%= @syslog_host %> - <%- end -%> - paths: - <%- @paths.each do |log_path| -%> - - <%= log_path %> - <%- end -%> - <%- if @encoding -%> - encoding: <%= @encoding %> - <%- end -%> - <%- if @include_lines.length > 0 -%> - include_lines: - <%- @include_lines.each do |include_line| -%> - - '<%= include_line %>' - <%- end -%> - <%- end -%> - <%- if @exclude_lines.length > 0 -%> - exclude_lines: - <%- @exclude_lines.each do |exclude_line| -%> - - '<%= exclude_line %>' - <%- end -%> - <%- end -%> - <%- if @exclude_files.length > 0 -%> - exclude_files: - <%- @exclude_files.each do |exclude_file| -%> - - <%= exclude_file %> - <%- end -%> - <%- end -%> - <%- if @ignore_older -%> - ignore_older: <%= @ignore_older %> - <%- end -%> - <%- if @doc_type -%> - document_type: <%= @doc_type %> - <%- end -%> - <%- if @scan_frequency -%> - prospector: - scanner: - check_interval: <%= @scan_frequency %> - <%- end -%> - <%- if @harvester_buffer_size -%> - harvester_buffer_size: <%= @harvester_buffer_size %> - <%- end -%> - <%- if @max_bytes -%> - message_max_bytes: <%= @max_bytes %> - <%- end -%> - <%- if @symlinks -%> - symlinks: <%= @symlinks %> - <%- end -%> - <%- if @close_older -%> - close_older: <%= @close_older %> - <%- end -%> - <%- if @force_close_files -%> - force_close_files: <%= @force_close_files %> - <%- end -%> - - <%- if @json.length > 0 -%> - ### JSON configuration - json: - # Decode JSON options. Enable this if your logs are structured in JSON. - # JSON key on which to apply the line filtering and multiline settings. This key - # must be top level and its value must be string, otherwise it is ignored. If - # no text key is defined, the line filtering and multiline features cannot be used. - <%- if @json['message_key'] != nil-%> - message_key: '<%= @json['message_key'] %>' - <%- end -%> - - # By default, the decoded JSON is placed under a "json" key in the output document. - # If you enable this setting, the keys are copied top level in the output document. - <%- if @json['keys_under_root'] != nil -%> - keys_under_root: <%= @json['keys_under_root'] %> - <%- end -%> - - # If keys_under_root and this setting are enabled, then the values from the decoded - # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) - # in case of conflicts. - <%- if @json['overwrite_keys'] != nil -%> - overwrite_keys: <%= @json['overwrite_keys'] %> - <%- end -%> - - # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON - # unmarshaling errors or when a text key is defined in the configuration but cannot - # be used. - <%- if @json['add_error_key'] != nil -%> - add_error_key: <%= @json['add_error_key'] %> - <%- end -%> - <%- end -%> - - <%- if @multiline.length > 0 -%> - parsers: - - multiline: - <%- if @multiline['pattern'] -%> - pattern: '<%= @multiline['pattern'] %>' - <%- end -%> - <%- if @multiline['negate'] -%> - negate: <%= @multiline['negate'] %> - <%- end -%> - <%- if @multiline['match'] -%> - match: <%= @multiline['match'] %> - <%- end -%> - <%- if @multiline['max_lines'] -%> - max_lines: <%= @multiline['max_lines'] %> - <%- end -%> - <%- if @multiline['timeout'] -%> - timeout: <%= @multiline['timeout'] %> - <%- end -%> - <%- end -%> - tail_files: <%= @tail_files %> - - # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the - # original for harvesting but will report the symlink name as source. - #symlinks: false - - <%- if @backoff or @max_backoff -%> - backoff: - <%- if @backoff -%> - init: <%= @backoff %> - <%- end -%> - <%- if @max_backoff -%> - max: <%= @max_backoff %> - <%- end -%> - <%- end -%> - - # Experimental: Max number of harvesters that are started in parallel. - # Default is 0 which means unlimited - <%- if @harvester_limit -%> - harvester_limit: <%= @harvester_limit %> - <%- end -%> - - ### Harvester closing options - - # Close inactive closes the file handler after the predefined period. - # The period starts when the last line of the file was, not the file ModTime. - # Time strings like 2h (2 hours), 5m (5 minutes) can be used. - <%- if @close_inactive -%> - close_inactive: <%= @close_inactive %> - <%- end -%> - - # Close renamed closes a file handler when the file is renamed or rotated. - # Note: Potential data loss. Make sure to read and understand the docs for this option. - close_renamed: <%= @close_renamed %> - - # When enabling this option, a file handler is closed immediately in case a file can't be found - # any more. In case the file shows up again later, harvesting will continue at the last known position - # after scan_frequency. - close_removed: <%= @close_removed %> - - # Closes the file handler as soon as the harvesters reaches the end of the file. - # By default this option is disabled. - # Note: Potential data loss. Make sure to read and understand the docs for this option. - close_eof: <%= @close_eof %> - - ### State options - - # Files for the modification data is older then clean_inactive the state from the registry is removed - # By default this is disabled. - <%- if @clean_inactive -%> - clean_inactive: <%= @clean_inactive %> - <%- end -%> - - # Removes the state for file which cannot be found on disk anymore immediately - clean_removed: <%= @clean_removed %> - - # Close timeout closes the harvester after the predefined time. - # This is independent if the harvester did finish reading the file or not. - # By default this option is disabled. - # Note: Potential data loss. Make sure to read and understand the docs for this option. - <%- if @close_timeout -%> - close_timeout: <%= @close_timeout %> - <%- end -%> - <%- if @pipeline -%> - pipeline: <%= @pipeline %> - <%- end -%> - <%- if @fields.length > 0 -%> - fields: - <%- @fields.each_pair do |k, v| -%> - <%= k %>: <%= v %> - <%- end -%> - <%- end -%> - fields_under_root: <%= @fields_under_root %> -<%- if @ssl.length > 0 -%> - ssl: - <%- @ssl.each_pair do |k, v| -%> - <%= k %>: <%= v %> - <%- end -%> - <%- end -%> - <%- if @tags.length > 0 -%> - tags: - <%- @tags.each do |tag| -%> - - <%= tag %> - <%- end -%> - <%- end -%> - <%- if @processors.length > 0 -%> - processors: - <%- %><%= @processors.to_yaml.lines.drop(1).join.gsub(/^/, ' ') -%> - <%- end -%> -<%- end %> diff --git a/templates/input.yml.erb b/templates/input.yml.erb index add9859..ae1b400 100644 --- a/templates/input.yml.erb +++ b/templates/input.yml.erb @@ -3,6 +3,9 @@ <%- else -%> --- - type: <%= @input_type %> + <%- if @input_type =~ /(filestream|journald)/ -%> + id: <%= @name %> + <%- end -%> <%- if @input_type =~ /(tcp|udp)/ -%> host: <%= @host %> <%- if @max_message_size -%> @@ -21,6 +24,16 @@ <%- elsif @input_type == 'syslog' -%> protocol.<%= @syslog_protocol %>: host: <%= @syslog_host %> + <%- elsif @input_type == 'journald' -%> + <%- if @seek -%> + seek: <%= @seek %> + <%- end -%> + <%- if @include_matches.length > 0 -%> + include_matches: + <%- @include_matches.each do |match| -%> + - <%= match %> + <%- end -%> + <%- end -%> <%- else -%> paths: <%- @paths.each do |log_path| -%> @@ -53,15 +66,25 @@ <%- if @doc_type -%> document_type: <%= @doc_type %> <%- end -%> - <%- if @scan_frequency -%> +<%- if @scan_frequency -%> + <%- if @input_type == 'filestream' -%> + prospector: + scanner: + check_interval: <%= @scan_frequency %> + <%- else -%> scan_frequency: <%= @scan_frequency %> <%- end -%> +<%- end -%> <%- if @harvester_buffer_size -%> harvester_buffer_size: <%= @harvester_buffer_size %> <%- end -%> - <%- if @max_bytes -%> +<%- if @max_bytes -%> + <%- if @input_type == 'filestream' -%> + message_max_bytes: <%= @max_bytes %> + <%- else -%> max_bytes: <%= @max_bytes %> <%- end -%> +<%- end -%> <%- if @symlinks -%> symlinks: <%= @symlinks %> <%- end -%> @@ -104,7 +127,26 @@ <%- end -%> <%- end -%> - <%- if @multiline.length > 0 -%> +<%- if @multiline.length > 0 -%> + <%- if @input_type == 'filestream' -%> + parsers: + - multiline: + <%- if @multiline['pattern'] -%> + pattern: '<%= @multiline['pattern'] %>' + <%- end -%> + <%- if @multiline['negate'] -%> + negate: <%= @multiline['negate'] %> + <%- end -%> + <%- if @multiline['match'] -%> + match: <%= @multiline['match'] %> + <%- end -%> + <%- if @multiline['max_lines'] -%> + max_lines: <%= @multiline['max_lines'] %> + <%- end -%> + <%- if @multiline['timeout'] -%> + timeout: <%= @multiline['timeout'] %> + <%- end -%> + <% else %> multiline: <%- if @multiline['pattern'] -%> pattern: '<%= @multiline['pattern'] %>' @@ -122,6 +164,7 @@ timeout: <%= @multiline['timeout'] %> <%- end -%> <%- end -%> +<%- end -%> tail_files: <%= @tail_files %> # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the @@ -198,6 +241,14 @@ <%- end -%> <%- end -%> fields_under_root: <%= @fields_under_root %> +<%- if scope.function_versioncmp([@facts['filebeat_version'], '7.5']) > 0 -%> + <%- if @index -%> + index: <%= @index %> + <%- end -%> + <%- if @keep_null -%> + keep_null: <%= @keep_null %> + <%- end -%> +<%- end -%> <%- if @ssl.length > 0 -%> ssl: <%- @ssl.each_pair do |k, v| -%>