diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-mysql-percona-guardium/CHANGELOG.md new file mode 100644 index 000000000..652c5a142 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/CHANGELOG.md @@ -0,0 +1,13 @@ + +## 1.1.7 +- More formatted exception description is added. +- Support for all timezone. + +## 1.0.0 +- Stable version + +### Added +- Initial release, in parallel to Guardium v11.4. + + + diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/filter.conf b/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/filter.conf index 4494f29e4..bda7f0741 100644 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/filter.conf +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/filter.conf @@ -1,11 +1,13 @@ filter{ # For this to work, the Filebeat configuration on your data source should tag the events it is sending. if [type] == "filebeat" and "guc_filter_param_datasource_tag" in [tags] { + + mutate { add_field => {"minOff" => "%{[event][timezone]}" }} mutate { add_field => { "source_program" => "percona-audit" } } mutate { add_field => { "server_hostname" => "%{[host][name]}" } } mutate { add_field => { "server_ip" => "%{[host][ip][0]}" } } mutate { replace => { "message" => "percona-audit: %{message}" } } - + mysql_percona_filter {} # keep original event fields, for debugging @@ -18,4 +20,4 @@ filter{ } } } -} +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/logstash-filter-mysql_percona_filter.zip b/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/logstash-filter-mysql_percona_filter.zip index 93138c508..d65d6f978 100644 Binary files a/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/logstash-filter-mysql_percona_filter.zip and b/filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage/MysqlPercona/logstash-filter-mysql_percona_filter.zip differ diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/VERSION b/filter-plugin/logstash-filter-mysql-percona-guardium/VERSION index 238d6e882..2bf1ca5f5 100644 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/VERSION +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/VERSION @@ -1 +1 @@ -1.0.7 +1.1.7 diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/perconaFilebeat.conf b/filter-plugin/logstash-filter-mysql-percona-guardium/perconaFilebeat.conf index 6f6b0be89..b9e380a3b 100644 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/perconaFilebeat.conf +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/perconaFilebeat.conf @@ -14,6 +14,8 @@ filter { # For this to work, the Filebeat configuration on your data source should tag the events it is sending. if [type] == "filebeat" and "mysqlpercona" in [tags] { + + mutate { add_field => {"minOff" => "%{[event][timezone]}" }} mutate { add_field => { "source_program" => "percona-audit" } } mutate { add_field => { "client_hostname" => "%{[agent][hostname]}" } } mutate { add_field => { "server_hostname" => "%{[host][hostname]}" } } diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java b/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java index 9b956e99d..f27976e8d 100755 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java @@ -7,6 +7,8 @@ import co.elastic.logstash.api.FilterMatchListener; import co.elastic.logstash.api.LogstashPlugin; import co.elastic.logstash.api.PluginConfigSpec; +import java.time.ZoneId; +import java.time.format.DateTimeParseException; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -41,7 +43,7 @@ public class MySqlPerconaFilter implements Filter { private static final String MYSQL_AUDIT_START_SIGNAL = "percona-audit: "; public static final String DATA_PROTOCOL_STRING = "MySQL Percona audit"; public static final String UNKNOWN_STRING = ""; - public static final String SERVER_TYPE_STRING = "MySql"; + public static final String SERVER_TYPE_STRING = "PERCONA_MYSQL"; private static Logger log = LogManager.getLogger(MySqlPerconaFilter.class); @@ -151,7 +153,13 @@ public Collection filter(Collection events, FilterMatchListener ma record.setAppUserName(UNKNOWN_STRING); - Time time = getTime(getFieldAsString(audit_record, "timestamp", null)); + String minOff = "+00:00" ; + if(e.getField("minOff") != null) + { + minOff = e.getField("minOff").toString(); + } + Time time = getTime(getFieldAsString(audit_record, "timestamp", null),minOff); + record.setTime(time); record.setSessionLocator(parseSessionLocator(e, audit_record)); @@ -187,14 +195,29 @@ private static String getFieldAsString(JsonObject jsonObject, String fieldName, return jsonObject.get(fieldName).getAsString(); } - public static Time getTime(String dateString){ + public static Time getTime(String dateString, String timeZone){ if (dateString == null){ log.warn("DateString is null"); return new Time(0, 0, 0); } - ZonedDateTime date = ZonedDateTime.parse(dateString, DATE_TIME_FORMATTER); - long millis = date.toInstant().toEpochMilli(); - int minOffset = date.getOffset().getTotalSeconds()/60; + + ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); + if (timeZone != null) { + offset = ZoneOffset.of(timeZone); + } + + ZonedDateTime date; + try { + date = ZonedDateTime.parse(dateString, DATE_TIME_FORMATTER); + } + catch(DateTimeParseException e){ + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + date = ZonedDateTime.parse(dateString, formatter.withZone(offset)); + } + + ZonedDateTime zdt = date.withZoneSameInstant(offset); + long millis = zdt.toInstant().toEpochMilli(); + int minOffset = zdt.getOffset().getTotalSeconds() / 60; //int minDst = date.getOffset().getRules().isDaylightSavings(date.toInstant()) ? 60 : 0; return new Time(millis, minOffset, 0); } diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/src/test/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilterTest.java b/filter-plugin/logstash-filter-mysql-percona-guardium/src/test/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilterTest.java index 5c8d0a2fb..aac105e33 100755 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/src/test/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilterTest.java +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/src/test/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilterTest.java @@ -4,7 +4,11 @@ import co.elastic.logstash.api.Context; import co.elastic.logstash.api.Event; import co.elastic.logstash.api.FilterMatchListener; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; import org.junit.Assert; import org.junit.Test; import org.logstash.plugins.ConfigurationImpl; @@ -82,6 +86,9 @@ public void testParseMySqlPercona_Data1(){ "\t\t\t\t\t\t\t\t\t\t\t\t\"ip\":\"\",\n" + "\t\t\t\t\t\t\t\t\t\t\t\t\"db\":\"gdb\"}\n" + "\t\t\t\t\t\t\t\t\t}"; + String minOff = "+07:00"; + + Configuration config = new ConfigurationImpl(Collections.singletonMap("log_level", "debug")); Context context = new ContextImpl(null, null); MySqlPerconaFilter filter = new MySqlPerconaFilter("test-id", config, context); @@ -89,6 +96,7 @@ public void testParseMySqlPercona_Data1(){ Event e = new org.logstash.Event(); TestMatchListener matchListener = new TestMatchListener(); + e.setField("minOff", minOff); e.setField("message", mysql_message); Collection results = filter.filter(Collections.singletonList(e), matchListener); @@ -96,12 +104,16 @@ public void testParseMySqlPercona_Data1(){ Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); Assert.assertEquals(1, matchListener.getMatchCount()); System.out.println(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Record record = new Gson().fromJson((String)e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), Record.class); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); } @Test public void testParseMySqlPercona_Error(){ String error_message = "percona-audit: {\"audit_record\":{\"name\":\"Query\",\"record\":\"39_2021-02-03T18:54:45\",\"timestamp\":\"2021-02-03T18:55:57 UTC\",\"command_class\":\"select\",\"connection_id\":\"2\",\"status\":1146,\"sqltext\":\"select * from users\",\"user\":\"root[root] @ localhost []\",\"host\":\"localhost\",\"os_user\":\"\",\"ip\":\"\",\"db\":\"mysql\"}}"; + String minOff = "+04:00"; + Configuration config = new ConfigurationImpl(Collections.singletonMap("source", "message")); Context context = new ContextImpl(null, null); MySqlPerconaFilter filter = new MySqlPerconaFilter("test-id", config, context); @@ -110,6 +122,7 @@ public void testParseMySqlPercona_Error(){ TestMatchListener matchListener = new TestMatchListener(); e.setField("message", error_message); + e.setField("minOff", minOff); Collection results = filter.filter(Collections.singletonList(e), matchListener); Assert.assertEquals(1, results.size()); @@ -138,6 +151,26 @@ public void testParseMySqlPercona_AccessDenied(){ Assert.assertEquals(1, matchListener.getMatchCount()); System.out.println(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); } + + @Test + public void testParseMySqlPercona_Data2(){ + String mysql_message = "<14>Feb 12 07:18:30 dbqa09 percona-audit: {\"audit_record\":{\"name\":\"Query\",\"record\":\"106_1970-01-01T00:00:00\",\"timestamp\":\"2021-02-12T12:18:30Z\",\"command_class\":\"select\",\"connection_id\":\"12\",\"status\":0,\"sqltext\":\"select * from Products limit 99999\",\"user\":\"root[root] @ localhost []\",\"host\":\"localhost\",\"os_user\":\"\",\"ip\":\"\",\"db\":\"\"}}"; + Configuration config = new ConfigurationImpl(Collections.singletonMap("log_level", "debug")); + Context context = new ContextImpl(null, null); + MySqlPerconaFilter filter = new MySqlPerconaFilter("test-id", config, context); + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + + e.setField("message", mysql_message); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); + System.out.println(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + } + } class TestMatchListener implements FilterMatchListener {