Add support for Journald input (#302)

* add support for journald * fix syntax error * fix syntax error 2 * fix incorrect ordering of if statements in template * appease the test suite * update readme
pcfens · Sep 20, 2022 · 3db9d9a · 3db9d9a
1 parent fee8805
commit 3db9d9a
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 223 deletions.
diff --git a/README.md b/README.md
@@ -400,6 +400,10 @@ to fully understand what these parameters do.
     [See above](#multiline-logs). (default: {})
   - `host`: [String] Host and port used to read events for TCP or UDP plugin (default: localhost:9000)
   - `max_message_size`: [String] The maximum size of the message received over TCP or UDP (default: undef)
+  - `keep_null`: [Boolean] If this option is set to true, fields with null values will be published in the output document (default: undef)
+  - `include_matches`: [Array] Journald input only, A collection of filter expressions used to match fields. The format of the expression is field=value (default: [])
+  - `seek`: [Enum] Journald input only, The position to start reading the journal from (default: undef)
+  - `index`: [String] If present, this formatted string overrides the index for events from this input (for elasticsearch outputs), or sets the raw_index field of the event’s metadata (for other outputs) (default: undef)
 
 ## Limitations
 This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping

diff --git a/manifests/input.pp b/manifests/input.pp
@@ -51,19 +51,17 @@
   Array $processors                        = [],
   Boolean $pure_array                      = false,
   String $host                             = 'localhost:9000',
+  Boolean $keep_null                       = false,
+  Array[String] $include_matches           = [],
+  Optional[Enum['head', 'tail', 'cursor']] $seek = undef,
   Optional[String] $max_message_size       = undef,
+  Optional[String] $index                  = undef,
 ) {
 
-  if Integer($filebeat::major_version) < 8 {
-    if versioncmp($facts['filebeat_version'], '7.16') > 0 {
-      $input_template = 'filestream.yml.erb'
-    } elsif versioncmp($facts['filebeat_version'], '6') > 0 {
-      $input_template = 'input.yml.erb'
-    } else {
-      $input_template = 'prospector.yml.erb'
-    }
+  if versioncmp($facts['filebeat_version'], '6') > 0 {
+    $input_template = 'input.yml.erb'
   } else {
-    $input_template = 'filestream.yml.erb'
+    $input_template = 'prospector.yml.erb'
   }
 
   if 'filebeat_version' in $facts and $facts['filebeat_version'] != false {
@@ -119,7 +117,7 @@
       $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? {
         true    => undef,
         default => $filebeat::major_version ? {
-          '5'     => "/usr/local/sbin/filebeat -N -configtest -c %",
+          '5'     => '/usr/local/sbin/filebeat -N -configtest -c %',
           default => "/usr/local/sbin/filebeat -c ${filebeat::config_file} test config",
         },
       }

diff --git a/manifests/params.pp b/manifests/params.pp
@@ -169,13 +169,10 @@
     }
   }
 
-  if Integer($major_version) < 8 {
-    if versioncmp($facts['filebeat_version'], '7.16') > 0 {
-      $default_input_type = 'filestream'
-    } else {
-      $default_input_type = 'log'
-    }
-  } else {
+  # filestream input type added in 7.10, deprecated in 7.16
+  if versioncmp($facts['filebeat_version'], '7.10') > 0 {
     $default_input_type = 'filestream'
+  } else {
+    $default_input_type = 'log'
   }
 }
diff --git a/templates/filestream.yml.erb b/templates/filestream.yml.erb
diff --git a/templates/input.yml.erb b/templates/input.yml.erb
@@ -3,6 +3,9 @@
 <%- else -%>
 ---
 - type: <%= @input_type %>
+  <%- if @input_type =~ /(filestream|journald)/ -%>
+  id: <%= @name %>
+  <%- end -%>
   <%- if @input_type =~ /(tcp|udp)/ -%>
   host: <%= @host %>
   <%- if @max_message_size -%>
@@ -21,6 +24,16 @@
   <%- elsif @input_type == 'syslog' -%>
   protocol.<%= @syslog_protocol %>:
     host: <%= @syslog_host %>
+  <%- elsif @input_type == 'journald' -%>
+  <%- if @seek -%>
+  seek: <%= @seek %>
+  <%- end -%>
+  <%- if @include_matches.length > 0 -%>
+  include_matches:
+    <%- @include_matches.each do |match| -%>
+    - <%= match %>
+    <%- end -%>
+  <%- end -%>
   <%- else -%>
   paths:
   <%- @paths.each do |log_path| -%>
@@ -53,15 +66,25 @@
   <%- if @doc_type -%>
   document_type: <%= @doc_type %>
   <%- end -%>
-  <%- if @scan_frequency -%>
+<%- if @scan_frequency -%>
+  <%- if @input_type == 'filestream' -%>
+  prospector:
+     scanner:
+        check_interval: <%= @scan_frequency %>
+  <%- else -%>
   scan_frequency: <%= @scan_frequency %>
   <%- end -%>
+<%- end -%>
   <%- if @harvester_buffer_size -%>
   harvester_buffer_size: <%= @harvester_buffer_size %>
   <%- end -%>
-  <%- if @max_bytes -%>
+<%- if @max_bytes -%>
+  <%- if @input_type == 'filestream' -%>
+  message_max_bytes: <%= @max_bytes %>
+  <%- else -%>
   max_bytes: <%= @max_bytes %>
   <%- end -%>
+<%- end -%>
   <%- if @symlinks -%>
   symlinks: <%= @symlinks %>
   <%- end -%>
@@ -104,7 +127,26 @@
       <%- end -%>
   <%- end -%>
 
-  <%- if @multiline.length > 0 -%>
+<%- if @multiline.length > 0 -%>
+  <%- if @input_type == 'filestream' -%>
+  parsers:
+      - multiline:
+          <%- if @multiline['pattern'] -%>
+          pattern: '<%= @multiline['pattern'] %>'
+          <%- end -%>
+          <%- if @multiline['negate'] -%>
+          negate: <%= @multiline['negate'] %>
+          <%- end -%>
+          <%- if @multiline['match'] -%>
+          match: <%= @multiline['match'] %>
+          <%- end -%>
+          <%- if @multiline['max_lines'] -%>
+          max_lines: <%= @multiline['max_lines'] %>
+          <%- end -%>
+          <%- if @multiline['timeout'] -%>
+          timeout: <%= @multiline['timeout'] %>
+          <%- end -%>
+  <% else %>
   multiline:
     <%- if @multiline['pattern'] -%>
     pattern: '<%= @multiline['pattern'] %>'
@@ -122,6 +164,7 @@
     timeout: <%= @multiline['timeout'] %>
     <%- end -%>
   <%- end -%>
+<%- end -%>
   tail_files: <%= @tail_files %>
 
   # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the
@@ -198,6 +241,14 @@
     <%- end -%>
   <%- end -%>
   fields_under_root: <%= @fields_under_root %>
+<%- if scope.function_versioncmp([@facts['filebeat_version'], '7.5']) > 0 -%>
+  <%- if @index -%>
+  index: <%= @index %>
+  <%- end -%>
+  <%- if @keep_null -%>
+  keep_null: <%= @keep_null %>
+  <%- end -%>
+<%- end -%>
 <%- if @ssl.length > 0 -%>
   ssl:
     <%- @ssl.each_pair do |k, v| -%>