fixes from branch release-v1.6.3

build/verifiedUCPlugins_gi.txt

@@ -36,12 +36,10 @@ filter-plugin/logstash-filter-pubsub-firestore-guardium/gi-pubsub-firestore-pack
 filter-plugin/logstash-filter-pubsub-mysql-guardium/gi-pubsub-mysql-package
 filter-plugin/logstash-filter-pubsub-bigquery-guardium/gi-pubsub-bigquery-package
 filter-plugin/logstash-filter-pubsub-bigtable-guardium/gi-pubsub-bigtable-package
-filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gi-pubsub-apachesolr-package
+filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gi-pubsub-apachsolr-package
 #Syslog
 filter-plugin/logstash-filter-onPremPostgres-guardium/PostgresOverSyslogPackage
 filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage
-filter-plugin/logstash-filter-mysql-guardium/MySQLOverSyslogPackage
-filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage
 #Other
 filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage
 filter-plugin/logstash-filter-azure-postgresql-guardium/AzurePostgresqlOverAzureEventHub

build/verified_UC_plugins_full_list.txt

@@ -38,16 +38,14 @@ filter-plugin/logstash-filter-pubsub-firestore-guardium/PubSubFireStorePackage;l
 filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage;logstash-filter-pubsub-mysql-guardium.zip
 filter-plugin/logstash-filter-pubsub-postgresql-guardium/PubSubPostgreSQLPackage;logstash-filter-pubsub-postgresql-guardium.zip
 #Pubsub plug-ins
-filter-plugin/logstash-filter-pubsub-spanner-guardium/gi-pubsub-spanner-package;spannerFilterPluginGIPackage.zip
-filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/gi-pubsub-firebase-package;PubSubFirebaseFilterPluginGIPackage.zip
-filter-plugin/logstash-filter-pubsub-firestore-guardium/gi-pubsub-firestore-package;firestoreFilterPluginGIPackage.zip
-filter-plugin/logstash-filter-pubsub-mysql-guardium/gi-pubsub-mysql-package;mysqlFilterPluginGIPackage.zip
-filter-plugin/logstash-filter-pubsub-bigquery-guardium/gi-pubsub-bigquery-package;bigQueryFilterPluginGIPackage.zip
-filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gi-pubsub-apachesolr-package;apachesolrFilterPluginGIPackage.zip
-filter-plugin/logstash-filter-pubsub-bigtable-guardium/gi-pubsub-bigtable-package;bigTableFilterPluginGIPackage.zip
+filter-plugin/logstash-filter-pubsub-spanner-guardium/gi-pubsub-spanner-package;logstash-filter-spanner_db_guardium_filter.zip
+filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/gi-pubsub-firebase-package;logstash-filter-fire_base_guardium_filter.zip
+filter-plugin/logstash-filter-pubsub-firestore-guardium/gi-pubsub-firestore-package;logstash-filter-fire_store_guardium_filter.zip
+filter-plugin/logstash-filter-pubsub-mysql-guardium/gi-pubsub-mysql-package;logstash-filter-pubsub-mysql-guardium.zip
+filter-plugin/logstash-filter-pubsub-bigquery-guardium/gi-pubsub-bigquery-package;logstash-filter-big_query_guardium_filter.zip
+filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gi-pubsub-apachsolr-package;logstash-filter-apache_solr_gcp_connector.zip
+filter-plugin/logstash-filter-pubsub-bigtable-guardium/gi-pubsub-bigtable-package;logstash-filter-big_table_guardium_filter.zip
 #Syslog plug-ins
-filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage;logstash-filter-mongodb_guardium_filter.zip
-filter-plugin/logstash-filter-mysql-guardium/MySQLOverSyslogPackage;logstash-filter-mysql_filter_guardium.zip
 filter-plugin/logstash-filter-onPremPostgres-guardium/PostgresOverSyslogPackage;
 filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage;
 #Other

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParser.java

@@ -26,8 +26,9 @@ public abstract class CustomParser {
     protected Map<String, String> properties;
     private final ObjectMapper mapper;
     private final IParser parser;
-    boolean parseUsingSniffer = false;
-    boolean hasSqlParsing = false;
+    protected boolean parseUsingSniffer = false;
+    protected boolean hasSqlParsing = false;
+    protected boolean parseUsingRegex = false;
 
     public CustomParser(ParserFactory.ParserType parserType) {
         parser = new ParserFactory().getParser(parserType);
@@ -40,9 +41,6 @@ public Record parseRecord(String payload) {
         if (!isValid(payload))
             return null;
 
-        hasSqlParsing = SqlParser.hasSqlParsing(properties);
-        parseUsingSniffer = hasSqlParsing && SqlParser.isSnifferParsing(payload);
-
         return extractRecord(payload);
     }
 
@@ -58,7 +56,7 @@ private Record extractRecord(String payload) {
         record.setSessionLocator(getSessionLocator(payload, record.getSessionId()));
         record.setTime(getTimestamp(payload));
 
-        if (record.isException())
+        if (!record.isException())
             record.setData(getData(payload, sqlString));
 
         return record;
@@ -409,7 +407,11 @@ protected boolean isValid(String payload) {
             return false;
         }
 
-        SqlParser.ValidityCase isValid = SqlParser.isValid(properties);
+        hasSqlParsing = SqlParser.hasSqlParsing(properties);
+        parseUsingSniffer = hasSqlParsing && SqlParser.isSnifferParsing(properties);
+        parseUsingRegex = hasSqlParsing && SqlParser.isRegexParsing(properties);
+
+        SqlParser.ValidityCase isValid = SqlParser.isValid(properties, hasSqlParsing, parseUsingSniffer, parseUsingRegex);
         if (!isValid.equals(SqlParser.ValidityCase.VALID)) {
             logger.error(isValid.getDescription());
             return false;

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/SqlParser.java

@@ -5,7 +5,7 @@
 import static com.ibm.guardium.universalconnector.commons.custom_parsing.PropertyConstant.*;
 
 public class SqlParser {
-    static List<String> validParsers = new ArrayList<>(Arrays.asList("REGEX", "SNIFFER"));
+    static List<String> validParsers = new ArrayList<>(Arrays.asList("REGEX", "SNIFFER", "JAVA"));
     static Map<String, String> validSnifferParsers;
 
     static {
@@ -36,24 +36,23 @@ public class SqlParser {
         validSnifferParsers = Collections.unmodifiableMap(map);
     }
 
-    public static String getServerType(String language) {
+    static String getServerType(String language) {
         return validSnifferParsers.get(language);
     }
 
-    public static ValidityCase isValid(Map<String, String> properties) {
-        boolean active = hasSqlParsing(properties);
-        if (!active)
+    static ValidityCase isValid(Map<String, String> properties, boolean hasSqlParsing, boolean parseUsingSniffer, boolean parseUsingRegex){
+        if (!hasSqlParsing)
             return ValidityCase.VALID;
 
         String parsingType = properties.get(PARSING_TYPE);
         if (parsingType == null || !validParsers.contains(parsingType))
             return ValidityCase.INVALID_PARSING_TYPE;
 
-        if (isSnifferParsing(parsingType)) {
+        if (parseUsingSniffer) {
             String snifferParser = properties.get(SNIFFER_PARSER);
             if (snifferParser == null || !validSnifferParsers.containsKey(snifferParser))
                 return ValidityCase.INVALID_SNIFFER_PARSER;
-        } else {
+        } else if(parseUsingRegex){
             String object = properties.get(OBJECT);
             if (object == null || object.isEmpty())
                 return ValidityCase.NULL_OBJECT;
@@ -66,15 +65,21 @@ public static ValidityCase isValid(Map<String, String> properties) {
         return ValidityCase.VALID;
     }
 
-    public static boolean hasSqlParsing(Map<String, String> properties) {
+    static boolean hasSqlParsing(Map<String, String> properties) {
         return Boolean.parseBoolean(properties.get(SQL_PARSING_ACTIVE));
     }
 
-    public static boolean isSnifferParsing(String parsingType) {
-        return parsingType.equalsIgnoreCase("SNIFFER");
+    static boolean isSnifferParsing(Map<String, String> properties) {
+        String parsingType = properties.get(PARSING_TYPE);
+        return parsingType!= null && parsingType.equalsIgnoreCase("SNIFFER");
+    }
+
+    static boolean isRegexParsing(Map<String, String> properties) {
+        String parsingType = properties.get(PARSING_TYPE);
+        return parsingType!= null && parsingType.equalsIgnoreCase("REGEX");
     }
 
-    public enum ValidityCase {
+    enum ValidityCase {
         VALID("The SQL Parsing is valid"),
         INVALID_PARSING_TYPE("Parsing type can only be REGEX or SNIFFER"),
         INVALID_SNIFFER_PARSER("Sniffer Parser is invalid."),

common/src/test/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParserTest.java

@@ -17,7 +17,6 @@
 import static org.junit.Assert.*;
 
 public class CustomParserTest {
-
     private static CustomParser customParser;
     private static Map<String, String> configValues;
 
@@ -315,38 +314,38 @@ public void testIsValidAndRelatedMethods() {
         Map<String, String> properties = new HashMap<>();
         properties.put(SQL_PARSING_ACTIVE, "false");
         assertFalse(SqlParser.hasSqlParsing(properties));
-        assertEquals(SqlParser.ValidityCase.VALID, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.VALID, SqlParser.isValid(properties, false, false, false));
 
         // Test hasSqlParsing when SQL parsing is active
         properties.put(SQL_PARSING_ACTIVE, "true");
         assertTrue(SqlParser.hasSqlParsing(properties));
 
         // Test isValid with invalid parsing type
         properties.put(PARSING_TYPE, "INVALID_TYPE");
-        assertEquals(SqlParser.ValidityCase.INVALID_PARSING_TYPE, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.INVALID_PARSING_TYPE, SqlParser.isValid(properties, true, false, false));
 
         // Test isValid with valid REGEX parsing type but null object
         properties.put(PARSING_TYPE, "REGEX");
         properties.put(VERB, "SELECT");
-        assertEquals(SqlParser.ValidityCase.NULL_OBJECT, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.NULL_OBJECT, SqlParser.isValid(properties, true, false, true));
 
         // Test isValid with valid REGEX parsing type but null verb
         properties.put(OBJECT, "table");
         properties.remove(VERB);
-        assertEquals(SqlParser.ValidityCase.NULL_VERB, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.NULL_VERB, SqlParser.isValid(properties, true, false, true));
 
         // Test isValid with valid REGEX parsing type and both object and verb present
         properties.put(VERB, "SELECT");
-        assertEquals(SqlParser.ValidityCase.VALID, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.VALID, SqlParser.isValid(properties, true, false, true));
 
         // Test isValid with valid SNIFFER parsing type but null sniffer parser
         properties.put(PARSING_TYPE, "SNIFFER");
         properties.remove(SNIFFER_PARSER);
-        assertEquals(SqlParser.ValidityCase.INVALID_SNIFFER_PARSER, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.INVALID_SNIFFER_PARSER, SqlParser.isValid(properties, true, true, false));
 
         // Test isValid with valid SNIFFER parsing type and valid sniffer parser
         properties.put(SNIFFER_PARSER, "MSSQL");
-        assertEquals(SqlParser.ValidityCase.VALID, SqlParser.isValid(properties));
+        assertEquals(SqlParser.ValidityCase.VALID, SqlParser.isValid(properties, true, true, false));
     }
 
     @Test
@@ -356,9 +355,19 @@ public void testGetServerTypeAndIsSnifferParsing() {
         assertNull(SqlParser.getServerType("INVALID_LANGUAGE"));
 
         // Test isSnifferParsing with valid and invalid parsing types
-        assertTrue(SqlParser.isSnifferParsing("SNIFFER"));
-        assertFalse(SqlParser.isSnifferParsing("REGEX"));
-        assertFalse(SqlParser.isSnifferParsing("INVALID_TYPE"));
+        Map<String, String> map = new HashMap<>();
+        map.put("parsing_type", "SNIFFER");
+        assertTrue(SqlParser.isSnifferParsing(map));
+
+        map.put("parsing_type", "REGEX");
+        assertFalse(SqlParser.isSnifferParsing(map));
+
+        map.put("parsing_type", "JAVA");
+        assertFalse(SqlParser.isSnifferParsing(map));
+
+        map.put("parsing_type", "something else");
+        assertFalse(SqlParser.isSnifferParsing(map));
+
         // Test getDescription for each ValidityCase
         assertEquals("The SQL Parsing is valid", SqlParser.ValidityCase.VALID.getDescription());
         assertEquals("Parsing type can only be REGEX or SNIFFER",

docs/available_plugins.md

@@ -23,8 +23,6 @@ To see if a particular plug-in is supported by older versions, please refer to t
 * [Elasticsearch](../filter-plugin/logstash-filter-elasticsearch-guardium/README.md)
 * [Google Cloud Apache Solr](../filter-plugin/logstash-filter-pubsub-apachesolr-guardium/README.md)
 * [Google Cloud BigQuery](../filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md) 
-* [Google Cloud Apache Solr](../filter-plugin/logstash-filter-pubsub-apachesolr-guardium/README.md) (Future GI releases)
-* [Google Cloud BigQuery](../filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md)
 * [Google Cloud BigTable](../filter-plugin/logstash-filter-pubsub-bigtable-guardium/README.md)
 * [Google Cloud Firebase](../filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/README.md) 
 * [Google Cloud Firestore](../filter-plugin/logstash-filter-pubsub-firestore-guardium/README.md) 

filter-plugin/logstash-filter-azure-postgresql-guardium/src/test/java/com/ibm/guardium/azurepostgresql/ParserTest.java

@@ -107,16 +107,6 @@ public void deleteQuery() throws ParseException {
 		Assert.assertEquals(record.getData().getOriginalSqlCommand(),"DELETE FROM Employee WHERE EmployeeNo = 101;");
 	}
 
-	@Test
-	public void testParseTimestamp() throws ParseException {
-
-		Event e = intitalizeEventObject();
-
-		e.setField(Constants.TIMESTAMP, "2022-03-02 14:06:56");
-		Time time = Parser.parseTimestamp(e);
-		Assert.assertEquals(1646210216000L, time.getTimstamp());
-	}
-
 	@Test
 	public void testParseSessionLocator() throws ParseException {
 		Event e = intitalizeEventObject();

filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/AWSMSSQL_README.md

@@ -156,20 +156,30 @@
 			VI. Click on Ok button.
 			VII. Right click on database audit specification that we have created and select enable to enable it.
 
-	5. Create non-admin user to access audit table.
-		If you want to access audit table without exposing admin credentials, create a non-admin user with specific permissions:
-
-			a. Log in into database by using the admin credentials and run the following queries.
-				CREATE LOGIN <login_name> WITH PASSWORD = '<password>';
-				USE msdb;
-				CREATE USER <user_name> FOR LOGIN <login_name>;
-				GRANT SELECT ON msdb.dbo.rds_fn_get_audit_file TO <user_name>;
-			b. In the input section, add the database name as 'msdb'
-      				jdbc_connection_string => "jdbc:sqlserver://<SERVER_NAME>:<PORT>;databaseName=<DB_NAME>;
-			c. Use the login credentials created in the previous step as the jdbc_username and password.
-					jdbc_user => "<login_name>"
-					jdbc_password => "<password>"
-
+### **Note: Create non-admin user to access audit table**
+
+To access the audit table without exposing admin credentials, create a non-admin user with specific permissions:
+
+- Log in to the database using admin credentials and run the following queries:
+  ```sql
+  CREATE LOGIN <login_name> WITH PASSWORD = '<password>';
+  USE msdb;
+  CREATE USER <user_name> FOR LOGIN <login_name>;
+  GRANT SELECT ON msdb.dbo.rds_fn_get_audit_file TO <user_name>;
+  ```
+
+- In the input section, set the database name as **msdb**.
+  ```properties
+  jdbc_connection_string => "jdbc:sqlserver://<SERVER_NAME>:<PORT>;databaseName=msdb;"
+  ```
+
+- Use the login credentials created in the previous step for the JDBC connection:
+  ```properties
+  jdbc_user => "<login_name>"
+  jdbc_password => "<password>"
+  ```
+
+- Update the input section by adding the details from the [awsNonAdminMSSQL.conf](./awsNonAdminMSSQL.conf) AWS MSSQL setup file, omitting the `input {` at the beginning and its corresponding `}` at the end.
 
 ## 3. Configuring the MSSQL filters in Guardium