-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
618 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,21 @@ | ||
| AuroraMysqlOverCloudwatchLogsProfile | ||
| AwsMssqlOverJdbcProfile | ||
| AzurePostgresOverEventHubProfile | ||
| CassandraOverFilebeatProfile | ||
| CouchbasedbOverFilebeatProfile | ||
| CouchdbOverFilebeatProfile | ||
| DynamoOverCloudwatchLogsProfile | ||
| FirebaseOverGooglePubSubProfile | ||
| GreenplumdbOverFilebeatProfile | ||
| MariaDBOverCloudwatchLogsProfile | ||
| MariaDBOverFilebeatProfile | ||
| MongoOverFilebeatProfile | ||
| MySQLOverCloudwatchLogsProfile | ||
| MysqlOverFilebeatProfile | ||
| MysqlPerconaOverFilebeatProfile | ||
| NeodbOverFilebeatProfile | ||
| PostgresOverCloudwatchLogsProfile | ||
| PostgresOverFilebeatProfile | ||
| PostgresOverKafkaProfile | ||
| PostgresOverSQSProfile | ||
| S3OverCloudwatchLogsProfile | ||
| S3OverCloudwatchLogsProfile |
32 changes: 32 additions & 0 deletions
32
gdp-packages/profile/MariaDBOverCloudwatchLogsProfile/filter/filter.conf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| filter{ | ||
| if [type] == "profile_name" { | ||
| # User can modify below filter to ignore system generated logs. Example: [message] =~ /(?i)select 1/ | ||
| if [message] =~ /(?i)select 1/ or [message] =~ /(?i)set autocommit/ or [message] =~ /(?i)SHOW INDEX FROM/ or [message] =~ /(?i)set @@SQL_SELECT_LIMIT/ or [message] =~ /(?i)SELECT @@tx_isolation/ or [message] =~ /(?i)select @@lower_case_table_names/ or [message] =~ /(?i)COMMIT/ or [message] =~ /(?i)ROLLBACK/ or [message] =~ /(?i)USE `/ or [message] =~ /(?i)SHOW FULL/ or [message] =~ /(?i)SHOW PROCEDURE STATUS/ or [message] =~ /(?i)SHOW FUNCTION STATUS/ or [message] =~ /(?i)SELECT DATABASE()/ or [message] =~ /(?i)use / or [message] =~ /(?i)SHOW COLLATION/ or [message] =~ /(?i)SHOW WARNINGS/ or [message] =~ /(?i)select @@/ or [message] =~ /(?i)SET GLOBAL/ or [message] =~ /(?i)SET NAMES/ or [message] =~ /(?i)SET character_set_results/ or [message] =~ /(?i)SHOW SESSION VARIABLES LIKE/ or [message] =~ /(?i)SELECT current_user()/ | ||
| { | ||
| drop{} | ||
| } | ||
| mutate { gsub => ["message","\"","--DQ--"]} | ||
| mutate { gsub => ["message","[\\]'",'"']} | ||
| csv | ||
| { | ||
| quote_char => "'" | ||
| columns => ["timestamp","serverhost","username","hostname","connectionid","queryid","operation","database","object","retcode"] | ||
| } | ||
| mutate { gsub => ["object","\"","'"]} | ||
| mutate { gsub => ["object","--DQ--","\""]} | ||
| if [operation] =~ /(?i)QUERY/ and "0" == [retcode] and ![database] and [message] !~ /(?i)create database/ | ||
| { | ||
| drop{} | ||
| } | ||
| if [operation] =~ /(?i)FAILED_CONNECT/ and "0" == [queryid] and ![username] | ||
| { | ||
| drop{} | ||
| } | ||
| mutate | ||
| { | ||
| add_field=> { "serverHostNamePrefix" => "%{account_id}_%{[cloudwatch_logs][log_stream]}" } | ||
| add_field=> { "dbNamePrefix" => "%{account_id}:%{[cloudwatch_logs][log_stream]}" } | ||
| } | ||
| awsmariadb_guardium_filter{} | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
gdp-packages/profile/MariaDBOverCloudwatchLogsProfile/filter/manifest.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "name": "Guardium_MariaDB_filter", | ||
| "alias": "MariaDB", | ||
| "type": "filter", | ||
| "pipeline_type":"pull", | ||
| "distribute_to_many": "false", | ||
| "plugin_version": "1.0.15", | ||
| "datasourceTypes": ["MariaDB"], | ||
| "supported_input_plugins": ["Cloudwatch logs input"], | ||
| "developer": "IBM", | ||
| "license": "Apache2.0", | ||
| "description": "Parses Amazon MariaDB database events into Guardium.", | ||
| "configuration_notes": "", | ||
| "documentation_path": "https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-mariadb-aws-guardium/README.md" | ||
| } |
75 changes: 75 additions & 0 deletions
75
gdp-packages/profile/MariaDBOverCloudwatchLogsProfile/package.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| { | ||
| "plugin_name": "MariaDB over Cloudwatch Logs", | ||
| "help_link": "https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-mariadb-aws-guardium/README.md", | ||
| "credentials": [ | ||
| "AWS Credentials", | ||
| "AWS Role ARN" | ||
| ], | ||
| "input_name": "Cloudwatch logs input", | ||
| "input_parameters": [ | ||
| { | ||
| "region": { | ||
| "name": "region", | ||
| "type": "text", | ||
| "label": "AWS account region", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 1 | ||
| }, | ||
| "log_group": { | ||
| "type": "string_list", | ||
| "label": "log group", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 2 | ||
| }, | ||
| "start_position": { | ||
| "type": "text", | ||
| "default": "end", | ||
| "label": "start position", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 3 | ||
| }, | ||
| "interval": { | ||
| "type": "number", | ||
| "default": 5, | ||
| "label": "Interval in sec", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 4 | ||
| }, | ||
| "event_filter": { | ||
| "type": "text", | ||
| "label": "Event filter (optional)", | ||
| "optional": true, | ||
| "hidden": false, | ||
| "sequence": 5, | ||
| "exclusion_regex": "param_event_filter", | ||
| "exclusion_replacement": "" | ||
| }, | ||
| "prefix": { | ||
| "type": "boolean", | ||
| "default": false, | ||
| "label": "Log group prefix", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 6 | ||
| }, | ||
| "account_id": { | ||
| "type": "text", | ||
| "label": "Account id", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 7, | ||
| "exclusion_regex": "add_field => \\{ \"account_id\" => \"param_account_id\" \\}", | ||
| "exclusion_replacement": "" | ||
| } | ||
| } | ||
| ], | ||
| "filter_name": "Guardium_MariaDB_filter", | ||
| "filter_parameters": [], | ||
| "supported_platforms": [ | ||
| "AWS" | ||
| ] | ||
| } |
129 changes: 129 additions & 0 deletions
129
gdp-packages/profile/MySQLOverCloudwatchLogsProfile/filter/filter.conf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| filter { | ||
| if [type] == "profile_name" { | ||
| #QUERIES | ||
| if "QUERY" in [message] { | ||
| grok { | ||
| match => { | ||
| "message" => "(?<yy>\d{4})(?<mm>\d{2})(?<dd>\d{2})\s%{TIME:time}(,)(ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3},)%{USER:db_user}(,)%{IPV4:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,)(')*([']?)(?<originalSqlCommand>.*)([']+,[0-9]+)" | ||
| } | ||
| } | ||
| } | ||
| #CONNECT / DISCONNECT | ||
| else if "DISCONNECT" in [message] or "CONNECT" in [message] { | ||
| grok { | ||
| match => { | ||
| "message" => "([0-9]+\s)%{TIME}(,)(ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3},)(%{USER:db_user})?(,)%{IPV4:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,,0)(,)?([^,]*)" | ||
| } | ||
| } | ||
| } | ||
| # "Access denied" | ||
| else { | ||
| grok { | ||
| match => { | ||
| "message" => "%{TIMESTAMP_ISO8601}(\s)+(?<data>%{NUMBER}(\s)([^\s]+)(\s)+([^']*)(')%{WORD:db_user}%{GREEDYDATA})" | ||
| } | ||
| } | ||
| } | ||
| mutate {add_field => { "logGroup" => "%{[cloudwatch_logs][log_group]}" } } | ||
| grok { match => { "logGroup" => "(?<data11>[^\/]*)\/(?<data12>[^\/]*)\/(?<data13>[^\/]*)\/(?<data14>[^\/]*)\/(?<instance>[^\/]*)\/(?<data15>[^\/]*)" } } | ||
| mutate { | ||
| add_field => { | ||
| "sessionLocator" => {} | ||
| "time" => {} | ||
| "accessor" => {} | ||
| "data" => {} | ||
| "ExceptionRecord" => {} | ||
| "GuardRecord" => {} | ||
| "serverIp" => "0.0.0.0" | ||
| "serverPort" => "-1" | ||
| "clientPort" => "-1" | ||
| } | ||
| } | ||
| mutate { | ||
| rename => { | ||
| "serverIp" => "[GuardRecord][sessionLocator][serverIp]" | ||
| "serverPort" => "[GuardRecord][sessionLocator][serverPort]" | ||
| "clientPort" => "[GuardRecord][sessionLocator][clientPort]" | ||
| "clientIp" => "[GuardRecord][sessionLocator][clientIp]" | ||
| "db_user" => "[GuardRecord][accessor][dbUser]" | ||
| } | ||
| add_field => { | ||
| "[GuardRecord][sessionId]" => "%{[cloudwatch_logs][event_id]}" | ||
| "[GuardRecord][accessor][dbProtocol]" => "MYSQL" | ||
| "[GuardRecord][accessor][dataType]" => "TEXT" | ||
| "[GuardRecord][accessor][language]" => "MYSQL" | ||
| "[GuardRecord][accessor][serverType]" => "MySql" | ||
| "[GuardRecord][accessor][serverOS]" => "" | ||
| "[GuardRecord][accessor][clientOs]" => "" | ||
| "[GuardRecord][accessor][clientHostName]" => "" | ||
| "[GuardRecord][accessor][serverHostName]" => "%{account_id}_%{instance}" | ||
| "[GuardRecord][accessor][commProtocol]" => "" | ||
| "[GuardRecord][accessor][dbProtocolVersion]" => "" | ||
| "[GuardRecord][accessor][osUser]" => "" | ||
| "[GuardRecord][accessor][sourceProgram]" => "" | ||
| "[GuardRecord][accessor][client_mac]" => "" | ||
| "[GuardRecord][accessor][serverDescription]" => "" | ||
| "[GuardRecord][accessor][serviceName]" => "" | ||
| "[GuardRecord][time][minOffsetFromGMT]" => "0" | ||
| "[GuardRecord][time][minDst]" => "0" | ||
| "[GuardRecord][sessionLocator][isIpv6]" => "false" | ||
| } | ||
| } | ||
| mutate { add_field => { "myTimestamp" => "%{yy}-%{mm}-%{dd}T%{time}Z" } } | ||
|
|
||
|
|
||
| date{ | ||
| match => ["myTimestamp", "ISO8601"] | ||
| target => "finalTime" | ||
| } | ||
|
|
||
| ruby { code => "event.set('[GuardRecord][time][timstamp]', event.get('finalTime').to_i * 1000)" } | ||
|
|
||
| if [dbName] { | ||
| mutate {add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{dbName}" } } | ||
| } | ||
| else { | ||
| mutate {add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}" } } | ||
| } | ||
| #Exceptions | ||
| if "Access denied" in [message] { | ||
| mutate { | ||
| add_field => { | ||
| "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" | ||
| "[GuardRecord][exception][sqlString]" => "" | ||
| "[GuardRecord][exception][description]" => "%{data}" | ||
| } | ||
| } | ||
| } else if "QUERY" in [message] { | ||
| mutate { | ||
| gsub => ["originalSqlCommand", "'", "\""] | ||
| gsub => ["originalSqlCommand", "\\n", " "] | ||
| gsub => ["originalSqlCommand", "\\t", ""] | ||
| gsub => ["originalSqlCommand", "[\\]", ""] | ||
|
|
||
| add_field => { | ||
| "[GuardRecord][data][originalSqlCommand]" => "%{originalSqlCommand}" | ||
| } | ||
| } | ||
| } else if "DISCONNECT" in [message] or "CONNECT" in [message] { | ||
| mutate { | ||
| add_field => { | ||
| "[GuardRecord][data][originalSqlCommand]" => "%{operation}" | ||
| } | ||
| } | ||
| } | ||
| else { | ||
| drop {} | ||
| } | ||
| if "_grokparsefailure" in [tags] { | ||
| drop {} | ||
| } | ||
| mutate { | ||
| remove_field => [ "message", "object", "@version", "cloudwatch_logs", "operation", "type", "data","account_id", "data13", "instance", "dbName", "data14", "logGroup", "data15", "data12", "yy", "mm", "dd" , "myTimestamp" , "finalTime" , "@timestamp" ] | ||
| } | ||
| json_encode { | ||
| source => "[GuardRecord]" | ||
| target => "[GuardRecord]" | ||
| } | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
gdp-packages/profile/MySQLOverCloudwatchLogsProfile/filter/manifest.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "name": "Guardium_MySQL_filter", | ||
| "alias": "MySQL", | ||
| "type": "filter", | ||
| "pipeline_type":"pull", | ||
| "distribute_to_many": "false", | ||
| "plugin_version": "1.0.0", | ||
| "datasourceTypes": ["MySQL"], | ||
| "supported_input_plugins": ["Cloudwatch logs input"], | ||
| "developer": "IBM", | ||
| "license": "Apache2.0", | ||
| "description": "Parses Amazon MySQL database events into Guardium.", | ||
| "configuration_notes": "", | ||
| "documentation_path": "https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-mysql-aws-guardium/README.md" | ||
| } |
75 changes: 75 additions & 0 deletions
75
gdp-packages/profile/MySQLOverCloudwatchLogsProfile/package.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| { | ||
| "plugin_name": "MySQL over Cloudwatch Logs", | ||
| "help_link": "https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-mysql-aws-guardium/README.md", | ||
| "credentials": [ | ||
| "AWS Credentials", | ||
| "AWS Role ARN" | ||
| ], | ||
| "input_name": "Cloudwatch logs input", | ||
| "input_parameters": [ | ||
| { | ||
| "region": { | ||
| "name": "region", | ||
| "type": "text", | ||
| "label": "AWS account region", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 1 | ||
| }, | ||
| "log_group": { | ||
| "type": "string_list", | ||
| "label": "log group", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 2 | ||
| }, | ||
| "start_position": { | ||
| "type": "text", | ||
| "default": "end", | ||
| "label": "start position", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 3 | ||
| }, | ||
| "interval": { | ||
| "type": "number", | ||
| "default": 5, | ||
| "label": "Interval in sec", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 4 | ||
| }, | ||
| "event_filter": { | ||
| "type": "text", | ||
| "label": "Event filter (optional)", | ||
| "optional": true, | ||
| "hidden": false, | ||
| "sequence": 5, | ||
| "exclusion_regex": "param_event_filter", | ||
| "exclusion_replacement": "" | ||
| }, | ||
| "prefix": { | ||
| "type": "boolean", | ||
| "default": false, | ||
| "label": "Log group prefix", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 6 | ||
| }, | ||
| "account_id": { | ||
| "type": "text", | ||
| "label": "Account id", | ||
| "optional": false, | ||
| "hidden": false, | ||
| "sequence": 7, | ||
| "exclusion_regex": "add_field => \\{ \"account_id\" => \"param_account_id\" \\}", | ||
| "exclusion_replacement": "" | ||
| } | ||
| } | ||
| ], | ||
| "filter_name": "Guardium_MySQL_filter", | ||
| "filter_parameters": [], | ||
| "supported_platforms": [ | ||
| "AWS" | ||
| ] | ||
| } |
Oops, something went wrong.