move grok step to a separated pipeline

elastic · Oct 11, 2024 · fbfdbbe · fbfdbbe
1 parent 11e6655
commit fbfdbbe
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 22 deletions.
diff --git a/filebeat/module/system/auth/ingest/files.yml b/filebeat/module/system/auth/ingest/files.yml
@@ -18,18 +18,9 @@ processors:
         TIMESTAMP: (?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})
       patterns:
         - '^%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:%{SPACE}%{GREEDYMULTILINE:_temp.message}$'
-  - grok:
+  - pipeline:
       description: Grok specific auth messages.
-      tag: grok-specific-messages
-      field: _temp.message
-      ignore_missing: true
-      patterns:
-        - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?'
-        - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}'
-        - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}'
-        - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}'
-        - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}'
-        - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$'
+      name: '{< IngestPipeline "grok-auth-messages" >}'
       on_failure:
         - rename:
             description: Leave the unmatched content in message.

diff --git a/filebeat/module/system/auth/ingest/grok-auth-messages.yml b/filebeat/module/system/auth/ingest/grok-auth-messages.yml
@@ -0,0 +1,14 @@
+description: Journald Pipeline for system/auth Filebeat module
+processors:
+  - grok:
+      description: Grok specific auth messages.
+      tag: grok-specific-messages
+      field: _temp.message
+      ignore_missing: true
+      patterns:
+        - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?'
+        - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}'
+        - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}'
+        - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}'
+        - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}'
+        - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$'
diff --git a/filebeat/module/system/auth/ingest/journald.yml b/filebeat/module/system/auth/ingest/journald.yml
@@ -6,19 +6,16 @@ processors:
   - rename:
       field: "journald.process.name"
       target_field: process.name
-  - grok:
-      description: Grok specific auth messages.
-      tag: grok-specific-messages
+  - rename:
       field: message
-      ignore_missing: true
-      patterns:
-        - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?'
-        - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}'
-        - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}'
-        - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}'
-        - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}'
-        - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$'
+      target_field: _temp.message
+  - pipeline:
+      description: Grok specific auth messages.
+      name: '{< IngestPipeline "grok-auth-messages" >}'
       ignore_failure: true
+  - rename:
+      field: _temp.message
+      target_field: message
   - grok:
       description: Grok usernames from PAM messages.
       tag: grok-pam-users

diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml
@@ -21,4 +21,5 @@ ingest_pipeline:
   - ingest/entrypoint.yml
   - ingest/files.yml
   - ingest/journald.yml
+  - ingest/grok-auth-messages.yml
 input: config/auth.yml