Merge pull request #718 from mlibrary/brandy-filebeat

Filebeat update

manifests/profile/elastic.pp

@@ -21,7 +21,7 @@
 
   apt::source { 'elastic.co':
     comment  => 'Elastic.co apt source for beats and elastic search',
-    location => 'https://artifacts.elastic.co/packages/5.x/apt',
+    location => 'https://artifacts.elastic.co/packages/7.x/apt',
     release  => 'stable',
     repos    => 'main',
     key      => {

manifests/profile/elastic/filebeat.pp

@@ -23,7 +23,7 @@
     notify  => Service['filebeat'],
   }
 
-  file { '/etc/filebeat/prospectors':
+  file { '/etc/filebeat/configs':
     ensure  => 'directory',
     require => Package['filebeat'],
   }

manifests/profile/elastic/filebeat/prospectors/ulib.pp → manifests/profile/elastic/filebeat/configs/ulib.pp

@@ -7,14 +7,14 @@
 # @summary A short summary of the purpose of this class
 #
 # @example
-#   include nebula::profile::elastic::filebeat::prospectors::ulib
-class nebula::profile::elastic::filebeat::prospectors::ulib (
+#   include nebula::profile::elastic::filebeat::configs::ulib
+class nebula::profile::elastic::filebeat::configs::ulib (
   Array $files = [],
 ) {
   include nebula::profile::elastic::filebeat
 
-  file { '/etc/filebeat/prospectors/ulib.yml':
-    content => template('nebula/profile/elastic/filebeat/prospectors/ulib.yml.erb'),
+  file { '/etc/filebeat/configs/ulib.yml':
+    content => template('nebula/profile/elastic/filebeat/configs/ulib.yml.erb'),
     notify  => Service['filebeat'],
   }
 }

manifests/role/aws.pp

@@ -30,6 +30,6 @@
 
   include nebula::profile::dns::aws
   include nebula::profile::elastic::metricbeat
-  include nebula::profile::elastic::filebeat::prospectors::ulib
+  include nebula::profile::elastic::filebeat::configs::ulib
 
 }

manifests/role/hathitrust.pp

@@ -35,6 +35,6 @@
 
   include nebula::profile::dns::smartconnect
   include nebula::profile::elastic::metricbeat
-  include nebula::profile::elastic::filebeat::prospectors::ulib
+  include nebula::profile::elastic::filebeat::configs::ulib
 
 }

manifests/role/mgetit_log.pp

@@ -8,7 +8,6 @@
 #   include nebula::role::mgetit_log
 class nebula::role::mgetit_log {
   include nebula::role::umich
-  include nebula::profile::elastic::filebeat::prospectors::mgetit
   include nebula::profile::nodejs
   include nebula::profile::php73
   include nebula::profile::ruby

manifests/role/umich.pp

@@ -29,6 +29,6 @@
 
   include nebula::profile::dns::standard
   include nebula::profile::elastic::metricbeat
-  include nebula::profile::elastic::filebeat::prospectors::ulib
+  include nebula::profile::elastic::filebeat::configs::ulib
 
 }

manifests/role/umich_mailserver.pp

@@ -29,6 +29,6 @@
 
   include nebula::profile::dns::standard
   include nebula::profile::elastic::metricbeat
-  include nebula::profile::elastic::filebeat::prospectors::ulib
+  include nebula::profile::elastic::filebeat::configs::ulib
 
 }

manifests/role/webhost/www_lib_vm.pp

@@ -12,7 +12,6 @@
   Hash $hosts = {}
 ) {
   include nebula::role::umich
-  include nebula::profile::elastic::filebeat::prospectors::clickstream
   include nebula::profile::www_lib::register_for_load_balancing
 
   class { 'nebula::profile::networking::private':

spec/classes/profile/elastic/filebeat/prospectors/ulib_spec.rb → spec/classes/profile/elastic/filebeat/configs/ulib_spec.rb

@@ -5,11 +5,11 @@
 # BSD License. See LICENSE.txt for details.
 require 'spec_helper'
 
-describe 'nebula::profile::elastic::filebeat::prospectors::ulib' do
+describe 'nebula::profile::elastic::filebeat::configs::ulib' do
   on_supported_os.each do |os, os_facts|
     context "on #{os}" do
       let(:facts) { os_facts }
-      let(:file) { '/etc/filebeat/prospectors/ulib.yml' }
+      let(:file) { '/etc/filebeat/configs/ulib.yml' }
 
       context 'with params' do
         let(:params) { { files: ['/var/log/1.log', '/var/log/logger/2.txt'] } }
@@ -19,7 +19,7 @@
         it do
           is_expected.to contain_file(file)
             .that_notifies('Service[filebeat]')
-            .with_content(%r{^\s+document_type: ulib$})
+            .with_content(%r{^\s+ulib_type: 90_day$})
         end
         it { is_expected.to contain_file(file).with_content(%r{^    \- "/var/log/1.log"$}) }
         it { is_expected.to contain_file(file).with_content(%r{^    \- "/var/log/logger/2.txt"$}) }

spec/classes/profile/elastic/filebeat_spec.rb

@@ -29,14 +29,14 @@
       end
 
       [
-        %r{^\s*config_dir: prospectors$},
+        %r{^\s*path: configs/\*.yml$},
         %r{^\s*hosts:.*"logstash.umdl.umich.edu:5044"},
       ].each do |content|
         it { is_expected.to contain_file('/etc/filebeat/filebeat.yml').with_content(content) }
       end
 
       it do
-        is_expected.to contain_file('/etc/filebeat/prospectors').with(
+        is_expected.to contain_file('/etc/filebeat/configs').with(
           ensure: 'directory',
           require: 'Package[filebeat]',
         )

spec/classes/profile/elastic_spec.rb

@@ -13,7 +13,7 @@
       it do
         is_expected.to contain_apt__source('elastic.co').with(
           comment: 'Elastic.co apt source for beats and elastic search',
-          location: 'https://artifacts.elastic.co/packages/5.x/apt',
+          location: 'https://artifacts.elastic.co/packages/7.x/apt',
           release: 'stable',
           repos: 'main',
           include: {

spec/fixtures/hiera/default.yaml

@@ -23,7 +23,6 @@ nebula::profile::dns::smartconnect::nameserver: 1.2.3.4
 nebula::profile::elastic::logstash_auth_cert: ''
 nebula::profile::elastic::logstash_hosts:
 - logstash.default.invalid:1234
-nebula::profile::elastic::filebeat::prospectors::mgetit::log_path: /var/log/mgetit.default.invalid
 nebula::profile::elastic::period: 90
 nebula::profile::haproxy::services: {}
 nebula::profile::haproxy::monitoring_user:

templates/profile/elastic/filebeat/configs/ulib.yml.erb

@@ -0,0 +1,12 @@
+# Managed by puppet (nebula/profile/elastic/filebeat/configs/ulib.yml.erb)
+- type: filestream
+  id: ulib
+  enabled: true
+  paths:
+  <% @files.each do |file| -%>
+  - "<%= file %>"
+  <% end %>
+
+  fields_under_root: true
+  fields:
+    ulib_type: 90_day

templates/profile/elastic/filebeat/filebeat.yml.erb

@@ -1,26 +1,30 @@
 # Managed by puppet (nebula/profile/elastic/filebeat/filebeat.yml.erb)
 
 ############################# Filebeat ######################################
-filebeat:
-  # All prospectors are in role- or service-specific files under prospectors/
-  config_dir: prospectors
-  registry_file: /var/lib/filebeat/registry
+filebeat.config.inputs:
+  enabled: true
+  path: configs/*.yml
+
+filebeat.registry.path: /var/lib/filebeat/registry
+
+filebeat.config.modules:
+  enabled: true
+  path: ${path.config}/modules.d/*.yml
 
 ############################# Output ##########################################
-output:
-  logstash:
+output.logstash:
     hosts: ["logstash.umdl.umich.edu:5044"]
     bulk_max_size: 1024
     ssl:
       certificate_authorities: ["/etc/ssl/certs/logstash-forwarder.crt"]
 
 ############################# Logging #########################################
+#
+logging.level: warning
 
-logging:
-  level: warning
-  files:
-    path: /var/log/filebeat
-    name: filebeat.log
-    keepfiles: 2
-    rotateeverybytes: 10485760 # = 10MB
-
+# Skip file-based logging config, given the documentation says:
+#
+# When Filebeat is running on a Linux system with systemd, it uses by default 
+# the -e command line option, that makes it write all the logging output to stderr 
+# so it can be captured by journald. Other outputs are disabled. See Filebeat 
+# and systemd to know more and learn how to change this.