From 613f9bce89967955345459077bd57b3a3e82e031 Mon Sep 17 00:00:00 2001 From: leweafan Date: Sun, 12 Mar 2023 23:13:10 +0300 Subject: [PATCH 01/11] Add MySQL authentication message parsing and related.ip & related.user fields --- CHANGELOG.next.asciidoc | 1 + .../module/mysql/error/ingest/pipeline.yml | 51 ++++++++++++- .../mysql/error/test/mariadb-10.4.8.log | 3 +- .../test/mariadb-10.4.8.log-expected.json | 73 +++++++++++++++++-- ...mysql-darwin-brew-5.7.10.log-expected.json | 6 +- .../test/mysql-ubuntu-5.5.53-extented.log | 1 + ...l-ubuntu-5.5.53-extented.log-expected.json | 25 +++++++ .../mysql-ubuntu-5.5.53.log-expected.json | 12 ++- .../mysql/error/test/mysql-ubuntu-8.0.15.log | 2 + .../mysql-ubuntu-8.0.15.log-expected.json | 46 ++++++++++++ .../module/mysql/slowlog/ingest/pipeline.json | 16 +++- .../test/mariadb-10.1.21.log-expected.json | 6 ++ .../test/mariadb-10.2.12.log-expected.json | 9 +++ .../test/mariadb-10.3.13.log-expected.json | 3 + .../test/mariadb-explain.log-expected.json | 3 + .../test/mysql-5.7.22.log-expected.json | 21 ++++++ ...mysql-darwin-brew-5.7.10.log-expected.json | 3 + .../mysql-debian-5.7.17.log-expected.json | 18 +++++ .../mysql-debian-5.7.19.log-expected.json | 6 ++ .../mysql-ubuntu-5.5.53.log-expected.json | 42 +++++++++++ .../mysql-ubuntu-8.0.15.log-expected.json | 6 ++ ...ona-ubuntu-5.7.19-innodb.log-expected.json | 9 +++ .../percona-ubuntu-5.7.19.log-expected.json | 36 +++++++++ .../percona-ubuntu-8.0.15.log-expected.json | 6 ++ 24 files changed, 391 insertions(+), 13 deletions(-) create mode 100644 filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log create mode 100644 filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7529c44acb7..83b2259c728 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -232,6 +232,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Add Basic Authentication support on constructed requests to CEL input {issue}34609[34609] {pull}34689[34689] - Add string manipulation extensions to CEL input {issue}34610[34610] {pull}34689[34689] - Add unix socket log parsing for nginx ingress_controller {pull}34732[34732] +- Add MySQL authentication message parsing and related.ip & related.user fields *Auditbeat* diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index ef3726e793d..460cb74a7ab 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -18,11 +18,16 @@ processors: - grok: field: message patterns: + - '(\[%{DATA:event.code}\])%{SPACE}(\[%{DATA:event.provider}\])%{SPACE}%{NOTSPACE}: Forcing close of thread %{INT} user: ''%{USERNAME:user.name}''' - '(\[%{DATA:event.code}\])%{SPACE}(\[%{DATA:event.provider}\])%{SPACE}%{GREEDYMULTILINE}' + - "%{ACCESS:_tmp.auth_failed} for user '%{USERNAME:user.name}'(@'%{IP:source.ip}')?" + - "IP address '%{IP:source.ip}' could not be resolved: Name or service not known" + - "Server socket created on IP: '%{IP:source.ip}'" - '%{GREEDYDATA}' ignore_missing: true ignore_failure: true pattern_definitions: + ACCESS: "Access denied" GREEDYMULTILINE: |- (.| )+ @@ -51,9 +56,6 @@ processors: field: _tmp.timestamp formats: - ISO8601 -- remove: - field: _tmp - ignore_missing: true - set: field: event.kind value: event @@ -67,6 +69,49 @@ processors: field: event.type value: error if: "ctx?.log?.level != null && ctx.log.level.toLowerCase() == 'error'" +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: ctx?.source?.ip != null +- append: + field: related.user + value: "{{user.name}}" + if: ctx?.user?.name != null +- append: + field: event.category + value: authentication + if: ctx?._tmp?.auth_failed != null +- append: + field: event.action + value: logon-failed + if: ctx?._tmp?.auth_failed != null +- set: + field: event.outcome + value: failure + if: ctx?._tmp?.auth_failed != null +- remove: + field: _tmp + ignore_missing: true on_failure: - set: field: error.message diff --git a/filebeat/module/mysql/error/test/mariadb-10.4.8.log b/filebeat/module/mysql/error/test/mariadb-10.4.8.log index 96bcaba1e7a..33623b283b8 100644 --- a/filebeat/module/mysql/error/test/mariadb-10.4.8.log +++ b/filebeat/module/mysql/error/test/mariadb-10.4.8.log @@ -27,4 +27,5 @@ Version: '10.4.8-MariaDB-log' socket: '/data/mysqldata/mysql.sock' port: 3306 2019-10-16 17:25:43 12 [Note] Event Scheduler: Dropping test.test_error_log 2019-10-16 17:25:43 12 [ERROR] Event Scheduler: [root@localhost][test.test_error_log] hi from the error log 2019-10-16 17:25:43 12 [Note] Event Scheduler: [root@localhost][test.test_error_log] At line 1 in test.test_error_log - +2023-03-02 17:00:06 200 [Warning] Access denied for user 'sherlock'@'localhost' (using password: NO) +2023-03-02 17:00:06 200 [Warning] Access denied for user 'sherlock'@'10.10.10.10' (using password: NO) diff --git a/filebeat/module/mysql/error/test/mariadb-10.4.8.log-expected.json b/filebeat/module/mysql/error/test/mariadb-10.4.8.log-expected.json index f5bc13d0ba9..3a24e8a34de 100644 --- a/filebeat/module/mysql/error/test/mariadb-10.4.8.log-expected.json +++ b/filebeat/module/mysql/error/test/mariadb-10.4.8.log-expected.json @@ -396,7 +396,11 @@ "log.offset": 1582, "message": "Server socket created on IP: '::'.", "mysql.thread_id": 0, - "service.type": "mysql" + "related.ip": [ + "::" + ], + "service.type": "mysql", + "source.ip": "::" }, { "@timestamp": "2019-10-16T17:24:15.000-02:00", @@ -556,13 +560,72 @@ ], "fileset.name": "error", "input.type": "log", - "log.flags": [ - "multiline" - ], "log.level": "Note", "log.offset": 2317, - "message": "Event Scheduler: [root@localhost][test.test_error_log] At line 1 in test.test_error_log\n", + "message": "Event Scheduler: [root@localhost][test.test_error_log] At line 1 in test.test_error_log", "mysql.thread_id": 12, "service.type": "mysql" + }, + { + "@timestamp": "2023-03-02T17:00:06.000-02:00", + "event.action": [ + "logon-failed" + ], + "event.category": [ + "authentication", + "database" + ], + "event.dataset": "mysql.error", + "event.kind": "event", + "event.module": "mysql", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 2435, + "message": "Access denied for user 'sherlock'@'localhost' (using password: NO)", + "mysql.thread_id": 200, + "related.user": [ + "sherlock" + ], + "service.type": "mysql", + "user.name": "sherlock" + }, + { + "@timestamp": "2023-03-02T17:00:06.000-02:00", + "event.action": [ + "logon-failed" + ], + "event.category": [ + "authentication", + "database" + ], + "event.dataset": "mysql.error", + "event.kind": "event", + "event.module": "mysql", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 2536, + "message": "Access denied for user 'sherlock'@'10.10.10.10' (using password: NO)", + "mysql.thread_id": 200, + "related.ip": [ + "10.10.10.10" + ], + "related.user": [ + "sherlock" + ], + "service.type": "mysql", + "source.ip": "10.10.10.10", + "user.name": "sherlock" } ] \ No newline at end of file diff --git a/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log-expected.json b/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log-expected.json index 3d64d000faa..933dde92804 100644 --- a/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log-expected.json +++ b/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log-expected.json @@ -775,7 +775,11 @@ "log.offset": 3746, "message": "Server socket created on IP: '::'.", "mysql.thread_id": 0, - "service.type": "mysql" + "related.ip": [ + "::" + ], + "service.type": "mysql", + "source.ip": "::" }, { "@timestamp": "2016-12-09T12:08:33.784Z", diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log new file mode 100644 index 00000000000..f93c528009d --- /dev/null +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log @@ -0,0 +1 @@ +230312 19:26:00 [Warning] IP address '10.10.10.10' could not be resolved: Name or service not known diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json new file mode 100644 index 00000000000..c305dd721bc --- /dev/null +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json @@ -0,0 +1,25 @@ +[ + { + "@timestamp": "2023-03-12T19:26:00.000-02:00", + "event.category": [ + "database" + ], + "event.dataset": "mysql.error", + "event.kind": "event", + "event.module": "mysql", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 0, + "message": "IP address '10.10.10.10' could not be resolved: Name or service not known", + "related.ip": [ + "10.10.10.10" + ], + "service.type": "mysql", + "source.ip": "10.10.10.10" + } +] \ No newline at end of file diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log-expected.json b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log-expected.json index 7bff59c06c3..2a08c2a1347 100644 --- a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log-expected.json +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log-expected.json @@ -1049,7 +1049,11 @@ "log.level": "Note", "log.offset": 4808, "message": "Server socket created on IP: '127.0.0.1'.", - "service.type": "mysql" + "related.ip": [ + "127.0.0.1" + ], + "service.type": "mysql", + "source.ip": "127.0.0.1" }, { "@timestamp": "2016-12-09T14:18:58.000-02:00", @@ -1428,7 +1432,11 @@ "log.level": "Note", "log.offset": 6305, "message": "Server socket created on IP: '127.0.0.1'.", - "service.type": "mysql" + "related.ip": [ + "127.0.0.1" + ], + "service.type": "mysql", + "source.ip": "127.0.0.1" }, { "@timestamp": "2016-12-09T14:37:58.000-02:00", diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log b/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log index 3b05f21997b..a598595fac5 100644 --- a/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log @@ -10,3 +10,5 @@ 2019-03-24T13:44:34.406962Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed. 2019-03-24T13:44:34.420123Z 0 [System] [MY-010931] [Server] /usr/sbin/mysqld: ready for connections. Version: '8.0.15' socket: '/var/run/mysqld/mysqld.sock' port: 3306 MySQL Community Server - GPL. 2019-03-24T13:44:34.572158Z 0 [System] [MY-011323] [Server] X Plugin ready for connections. Socket: '/var/run/mysqld/mysqlx.sock' bind-address: '::' port: 33060 +2023-03-12T15:30:16.866015Z 4016575 [Note] Aborted connection 4016575 to db: 'testdb' user: 'sherlock' host: 'localhost' (Got an error reading communication packets) +2023-03-07T09:34:58.461438Z 0 [Warning] [MY-010909] [Server] /usr/sbin/mysqld: Forcing close of thread 14 user: 'sherlock'. diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log-expected.json b/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log-expected.json index 3711000335a..78599265db2 100644 --- a/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log-expected.json +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log-expected.json @@ -262,5 +262,51 @@ "message": "[MY-011323] [Server] X Plugin ready for connections. Socket: '/var/run/mysqld/mysqlx.sock' bind-address: '::' port: 33060", "mysql.thread_id": 0, "service.type": "mysql" + }, + { + "@timestamp": "2023-03-12T15:30:16.866Z", + "event.category": [ + "database" + ], + "event.dataset": "mysql.error", + "event.kind": "event", + "event.module": "mysql", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "error", + "input.type": "log", + "log.level": "Note", + "log.offset": 1723, + "message": "Aborted connection 4016575 to db: 'testdb' user: 'sherlock' host: 'localhost' (Got an error reading communication packets)", + "mysql.thread_id": 4016575, + "service.type": "mysql" + }, + { + "@timestamp": "2023-03-07T09:34:58.461Z", + "event.category": [ + "database" + ], + "event.code": "MY-010909", + "event.dataset": "mysql.error", + "event.kind": "event", + "event.module": "mysql", + "event.provider": "Server", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 1889, + "message": "[MY-010909] [Server] /usr/sbin/mysqld: Forcing close of thread 14 user: 'sherlock'.", + "mysql.thread_id": 0, + "related.user": [ + "sherlock" + ], + "service.type": "mysql", + "user.name": "sherlock" } ] \ No newline at end of file diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 5bb43a217de..fa1dd7df9c0 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -94,6 +94,20 @@ "field": "event.type", "value": "info" } + }, + { + "append": { + "field": "related.user", + "value": "{{user.name}}", + "if": "ctx?.user?.name != null" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{source.ip}}", + "if": "ctx?.source?.ip != null" + } } ], "on_failure": [ @@ -104,4 +118,4 @@ } } ] -} \ No newline at end of file +} diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json index ca78d11e19c..cc634c20c8c 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json @@ -24,6 +24,12 @@ "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, "mysql.thread_id": "5", + "related.ip": [ + "121.0.0.1" + ], + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "source.ip": "121.0.0.1", diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json index aeae8bc94cd..f6204c959e0 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json @@ -25,6 +25,9 @@ "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, "mysql.thread_id": "8", + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" @@ -64,6 +67,12 @@ "mysql.slowlog.tmp_table": true, "mysql.slowlog.tmp_table_on_disk": false, "mysql.thread_id": "25844", + "related.ip": [ + "192.168.0.10" + ], + "related.user": [ + "root" + ], "service.type": "mysql", "source.ip": "192.168.0.10", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log-expected.json index 16cc4a6854f..25218a2ad0f 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log-expected.json @@ -38,6 +38,9 @@ "mysql.slowlog.tmp_table_sizes": 4026528, "mysql.slowlog.tmp_tables": 1, "mysql.thread_id": "37", + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json index 92e24e577bb..1fb80e70005 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json @@ -25,6 +25,9 @@ "mysql.slowlog.rows_sent": 1, "mysql.slowlog.schema": "dbt3sf1", "mysql.thread_id": "2", + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json index d5748da6547..6d30f214fc5 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json @@ -23,6 +23,12 @@ "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, "mysql.thread_id": 7234, + "related.ip": [ + "218.76.8.37" + ], + "related.user": [ + "root" + ], "service.type": "mysql", "source.ip": "218.76.8.37", "user.name": "root" @@ -50,6 +56,9 @@ "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", "mysql.slowlog.rows_examined": 5, "mysql.slowlog.rows_sent": 1, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -78,6 +87,12 @@ "mysql.slowlog.rows_examined": 1489615, "mysql.slowlog.rows_sent": 1000, "mysql.thread_id": 10997316, + "related.ip": [ + "1.1.1.1" + ], + "related.user": [ + "appuser" + ], "service.type": "mysql", "source.domain": "apphost", "source.ip": "1.1.1.1", @@ -107,6 +122,12 @@ "mysql.slowlog.rows_examined": 4751313, "mysql.slowlog.rows_sent": 0, "mysql.thread_id": 10999834, + "related.ip": [ + "1.1.1.1" + ], + "related.user": [ + "appuser" + ], "service.type": "mysql", "source.domain": "apphost", "source.ip": "1.1.1.1", diff --git a/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log-expected.json index a439aa4a5aa..e49d3144f7a 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log-expected.json @@ -23,6 +23,9 @@ "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, "mysql.thread_id": 2, + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.17.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.17.log-expected.json index bdbee3aad14..198ee1acc27 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.17.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.17.log-expected.json @@ -23,6 +23,12 @@ "mysql.slowlog.rows_examined": 1489615, "mysql.slowlog.rows_sent": 1000, "mysql.thread_id": 10997316, + "related.ip": [ + "1.1.1.1" + ], + "related.user": [ + "apphost" + ], "service.type": "mysql", "source.domain": "apphost", "source.ip": "1.1.1.1", @@ -52,6 +58,12 @@ "mysql.slowlog.rows_examined": 4751313, "mysql.slowlog.rows_sent": 0, "mysql.thread_id": 10999834, + "related.ip": [ + "1.1.1.1" + ], + "related.user": [ + "apphost" + ], "service.type": "mysql", "source.domain": "apphost", "source.ip": "1.1.1.1", @@ -81,6 +93,12 @@ "mysql.slowlog.rows_examined": 4754675, "mysql.slowlog.rows_sent": 0, "mysql.thread_id": 11004208, + "related.ip": [ + "1.1.1.1" + ], + "related.user": [ + "apphost" + ], "service.type": "mysql", "source.domain": "apphost", "source.ip": "1.1.1.1", diff --git a/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.19.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.19.log-expected.json index b6cba473929..0071d064534 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.19.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-debian-5.7.19.log-expected.json @@ -23,6 +23,12 @@ "mysql.slowlog.rows_examined": 101, "mysql.slowlog.rows_sent": 101, "mysql.thread_id": 5, + "related.ip": [ + "172.17.0.11" + ], + "related.user": [ + "root" + ], "service.type": "mysql", "source.ip": "172.17.0.11", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log-expected.json index 3e4b0853e28..d2a50e1663b 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log-expected.json @@ -22,6 +22,9 @@ "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", "mysql.slowlog.rows_examined": 5, "mysql.slowlog.rows_sent": 1, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -49,6 +52,9 @@ "mysql.slowlog.query": "select concat('select count(*) into @discard from `',\n TABLE_SCHEMA, '`.`', TABLE_NAME, '`')\n from information_schema.TABLES where ENGINE='MyISAM';", "mysql.slowlog.rows_examined": 81, "mysql.slowlog.rows_sent": 31, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -76,6 +82,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`COLUMNS`;", "mysql.slowlog.rows_examined": 808, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -103,6 +112,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`EVENTS`;", "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -130,6 +142,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`PARAMETERS`;", "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -157,6 +172,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`PARTITIONS`;", "mysql.slowlog.rows_examined": 81, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -184,6 +202,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`PLUGINS`;", "mysql.slowlog.rows_examined": 23, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -211,6 +232,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`PROCESSLIST`;", "mysql.slowlog.rows_examined": 1, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -238,6 +262,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`ROUTINES`;", "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -265,6 +292,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`TRIGGERS`;", "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -292,6 +322,9 @@ "mysql.slowlog.query": "select count(*) into @discard from `information_schema`.`VIEWS`;", "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 0, + "related.user": [ + "debian-sys-maint" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "debian-sys-maint" @@ -319,6 +352,9 @@ "mysql.slowlog.query": "select sleep(2);", "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" @@ -347,6 +383,9 @@ "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 0, "mysql.slowlog.schema": "mysql", + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" @@ -374,6 +413,9 @@ "mysql.slowlog.query": "select * from user;", "mysql.slowlog.rows_examined": 5, "mysql.slowlog.rows_sent": 5, + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json index 5e258b4c210..ed03467ea1d 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json @@ -24,6 +24,9 @@ "mysql.slowlog.rows_sent": 10, "mysql.slowlog.schema": "employees", "mysql.thread_id": 14, + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" @@ -72,6 +75,9 @@ "mysql.slowlog.tmp_disk_tables": 0, "mysql.slowlog.tmp_tables": 1, "mysql.thread_id": "16", + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json index 9f4cf24cb83..d0881b36247 100644 --- a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json @@ -40,6 +40,9 @@ "mysql.slowlog.tmp_table_sizes": 606208, "mysql.slowlog.tmp_tables": 376, "mysql.thread_id": 14367293, + "related.user": [ + "exporter" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "exporter" @@ -93,6 +96,12 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 0, "mysql.thread_id": 14370752, + "related.ip": [ + "192.168.123.123" + ], + "related.user": [ + "test" + ], "service.type": "mysql", "source.ip": "192.168.123.123", "user.name": "test" diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log-expected.json index fc6a7af2d25..1e3dc53da5b 100644 --- a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log-expected.json @@ -40,6 +40,9 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 1, "mysql.thread_id": 1098148226, + "related.user": [ + "check" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "check" @@ -92,6 +95,12 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 0, "mysql.thread_id": 1101779094, + "related.ip": [ + "192.168.123.123" + ], + "related.user": [ + "select" + ], "service.type": "mysql", "source.ip": "192.168.123.123", "user.name": "select" @@ -137,6 +146,9 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 0, "mysql.thread_id": 14366748, + "related.user": [ + "exporter" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "exporter" @@ -184,6 +196,12 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 0, "mysql.thread_id": 14349788, + "related.ip": [ + "192.168.123.123" + ], + "related.user": [ + "test" + ], "service.type": "mysql", "source.ip": "192.168.123.123", "user.name": "test" @@ -229,6 +247,9 @@ "mysql.slowlog.tmp_table_sizes": 114688, "mysql.slowlog.tmp_tables": 111, "mysql.thread_id": 14367106, + "related.user": [ + "exporter" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "exporter" @@ -274,6 +295,9 @@ "mysql.slowlog.tmp_table_sizes": 606208, "mysql.slowlog.tmp_tables": 376, "mysql.thread_id": 14367293, + "related.user": [ + "exporter" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "exporter" @@ -326,6 +350,12 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 0, "mysql.thread_id": 14360213, + "related.ip": [ + "192.168.123.123" + ], + "related.user": [ + "test" + ], "service.type": "mysql", "source.ip": "192.168.123.123", "user.name": "test" @@ -379,6 +409,12 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 0, "mysql.thread_id": 14370752, + "related.ip": [ + "192.168.123.123" + ], + "related.user": [ + "test" + ], "service.type": "mysql", "source.ip": "192.168.123.123", "user.name": "test" diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json index fdebbcef421..5cc4543a970 100644 --- a/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json @@ -28,6 +28,9 @@ "mysql.slowlog.rows_sent": 10, "mysql.slowlog.schema": "employees", "mysql.thread_id": 182, + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" @@ -78,6 +81,9 @@ "mysql.slowlog.tmp_table_sizes": 0, "mysql.slowlog.tmp_tables": 1, "mysql.thread_id": 8, + "related.user": [ + "root" + ], "service.type": "mysql", "source.domain": "localhost", "user.name": "root" From 0ea101135095c020e9bee7cb0bdf9ffd6f709626 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Sun, 12 Mar 2023 23:19:50 +0300 Subject: [PATCH 02/11] Added pull id --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 83b2259c728..71f56f46d63 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -232,7 +232,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Add Basic Authentication support on constructed requests to CEL input {issue}34609[34609] {pull}34689[34689] - Add string manipulation extensions to CEL input {issue}34610[34610] {pull}34689[34689] - Add unix socket log parsing for nginx ingress_controller {pull}34732[34732] -- Add MySQL authentication message parsing and related.ip & related.user fields +- Add MySQL authentication message parsing and related.ip & related.user fields {pull}34810[34810] *Auditbeat* From 21541862854d55f3eb7ff20db0c0095bcfa0c87e Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:16:42 +0300 Subject: [PATCH 03/11] Update filebeat/module/mysql/error/ingest/pipeline.yml Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/error/ingest/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index 460cb74a7ab..e4d95452312 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -92,7 +92,7 @@ processors: - append: field: related.ip value: "{{source.ip}}" - if: ctx?.source?.ip != null + if: ctx.source?.ip != null - append: field: related.user value: "{{user.name}}" From bb696ed18bb2d5b22dac946802e70d001acfcf48 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:16:50 +0300 Subject: [PATCH 04/11] Update filebeat/module/mysql/error/ingest/pipeline.yml Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/error/ingest/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index e4d95452312..672c3ce531c 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -96,7 +96,7 @@ processors: - append: field: related.user value: "{{user.name}}" - if: ctx?.user?.name != null + if: ctx.user?.name != null - append: field: event.category value: authentication From 0b199c7b14c61c667d5bdd46d72f312c0891a494 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:17:19 +0300 Subject: [PATCH 05/11] Update filebeat/module/mysql/error/ingest/pipeline.yml Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/error/ingest/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index 672c3ce531c..771a617a856 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -100,7 +100,7 @@ processors: - append: field: event.category value: authentication - if: ctx?._tmp?.auth_failed != null + if: ctx._tmp?.auth_failed != null - append: field: event.action value: logon-failed From be6221949ce3f952725cb610706a4d46ef19dce2 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:17:28 +0300 Subject: [PATCH 06/11] Update filebeat/module/mysql/error/ingest/pipeline.yml Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/error/ingest/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index 771a617a856..12f7296b326 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -104,7 +104,7 @@ processors: - append: field: event.action value: logon-failed - if: ctx?._tmp?.auth_failed != null + if: ctx._tmp?.auth_failed != null - set: field: event.outcome value: failure From 68c52df92af198a19effdbf79e85588bcab405c8 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:17:37 +0300 Subject: [PATCH 07/11] Update filebeat/module/mysql/error/ingest/pipeline.yml Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/error/ingest/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index 12f7296b326..c8648a3c116 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -108,7 +108,7 @@ processors: - set: field: event.outcome value: failure - if: ctx?._tmp?.auth_failed != null + if: ctx._tmp?.auth_failed != null - remove: field: _tmp ignore_missing: true From 902ee2e928987e16b4cf79b05401da2b8e71ccd0 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:17:51 +0300 Subject: [PATCH 08/11] Update filebeat/module/mysql/slowlog/ingest/pipeline.json Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/slowlog/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index fa1dd7df9c0..f93d67afd6b 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -99,7 +99,7 @@ "append": { "field": "related.user", "value": "{{user.name}}", - "if": "ctx?.user?.name != null" + "if": "ctx.user?.name != null" } }, { From 7a3c5e47dcad6bbaf905540947c6066b7ba85845 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:18:06 +0300 Subject: [PATCH 09/11] Update filebeat/module/mysql/slowlog/ingest/pipeline.json Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- filebeat/module/mysql/slowlog/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index f93d67afd6b..02060e8ecbf 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -106,7 +106,7 @@ "append": { "field": "related.ip", "value": "{{source.ip}}", - "if": "ctx?.source?.ip != null" + "if": "ctx.source?.ip != null" } } ], From 6428ede33153435be91c78bcfbd7fa3d953d385b Mon Sep 17 00:00:00 2001 From: Alexander A Date: Wed, 29 Mar 2023 00:23:34 +0300 Subject: [PATCH 10/11] Update CHANGELOG.next.asciidoc Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 71f56f46d63..65402f5a186 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -232,7 +232,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Add Basic Authentication support on constructed requests to CEL input {issue}34609[34609] {pull}34689[34689] - Add string manipulation extensions to CEL input {issue}34610[34610] {pull}34689[34689] - Add unix socket log parsing for nginx ingress_controller {pull}34732[34732] -- Add MySQL authentication message parsing and related.ip & related.user fields {pull}34810[34810] +- Add MySQL authentication message parsing and `related.ip` and `related.user` fields {pull}34810[34810] *Auditbeat* From c8b1624ac2b29788e0e20dbf93d679df8ae03ffd Mon Sep 17 00:00:00 2001 From: leweafan Date: Wed, 29 Mar 2023 01:09:09 +0300 Subject: [PATCH 11/11] updated pipeline and expected log file --- filebeat/module/mysql/error/ingest/pipeline.yml | 7 ++++++- .../test/mysql-ubuntu-5.5.53-extented.log-expected.json | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index c8648a3c116..122b6d274cf 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -21,13 +21,14 @@ processors: - '(\[%{DATA:event.code}\])%{SPACE}(\[%{DATA:event.provider}\])%{SPACE}%{NOTSPACE}: Forcing close of thread %{INT} user: ''%{USERNAME:user.name}''' - '(\[%{DATA:event.code}\])%{SPACE}(\[%{DATA:event.provider}\])%{SPACE}%{GREEDYMULTILINE}' - "%{ACCESS:_tmp.auth_failed} for user '%{USERNAME:user.name}'(@'%{IP:source.ip}')?" - - "IP address '%{IP:source.ip}' could not be resolved: Name or service not known" + - '%{IP_RESOLVE_ERROR:_tmp.ip_resolve_error}' - "Server socket created on IP: '%{IP:source.ip}'" - '%{GREEDYDATA}' ignore_missing: true ignore_failure: true pattern_definitions: ACCESS: "Access denied" + IP_RESOLVE_ERROR: "IP address '%{IP:source.ip}' could not be resolved: Name or service not known" GREEDYMULTILINE: |- (.| )+ @@ -101,6 +102,10 @@ processors: field: event.category value: authentication if: ctx._tmp?.auth_failed != null +- append: + field: event.category + value: network + if: ctx?._tmp?.ip_resolve_error != null - append: field: event.action value: logon-failed diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json index c305dd721bc..c15aebb49d4 100644 --- a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53-extented.log-expected.json @@ -2,7 +2,8 @@ { "@timestamp": "2023-03-12T19:26:00.000-02:00", "event.category": [ - "database" + "database", + "network" ], "event.dataset": "mysql.error", "event.kind": "event", @@ -22,4 +23,4 @@ "service.type": "mysql", "source.ip": "10.10.10.10" } -] \ No newline at end of file +]