Merge pull request #675 from IBM/INS-49047-v2

INS-49047: Adding JSON/Leef parser

common/build.gradle

@@ -32,6 +32,8 @@ dependencies {
     implementation group: 'commons-validator', name: 'commons-validator', version: '1.7'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.23.1'
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2'
+    implementation group: 'com.google.guava', name: 'guava', version: '33.3.1-jre'
+    implementation group: 'com.google.code.gson', name: 'gson', version: '2.11.0'
 
     testImplementation 'junit:junit:4.13.1'
     testImplementation 'org.mockito:mockito-core:3.12.4'

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParser.java

@@ -1,6 +1,8 @@
 package com.ibm.guardium.universalconnector.commons.custom_parsing;
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.ibm.guardium.universalconnector.commons.custom_parsing.excepton.InvalidConfigurationException;
 import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.IParser;
 import com.ibm.guardium.universalconnector.commons.structures.*;
 import org.apache.commons.validator.routines.InetAddressValidator;
@@ -25,26 +27,35 @@ public abstract class CustomParser {
     private static final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance();
     protected Map<String, String> properties;
     private final ObjectMapper mapper;
-    private final IParser parser;
+    IParser parser;
     protected boolean parseUsingSniffer = false;
     protected boolean hasSqlParsing = false;
-    protected boolean parseUsingRegex = false;
+    protected boolean parseUsingCustomParser = false;
 
     public CustomParser(ParserFactory.ParserType parserType) {
         parser = new ParserFactory().getParser(parserType);
         mapper = new ObjectMapper();
+
+        // We only need to read the properties file once and then we validate it.
+        try {
+            properties = getProperties();
+        } catch (InvalidConfigurationException e) {
+            logger.error("The config file is corrupted so the parser cannot parse the logs: ", e);
+        }
     }
 
     public Record parseRecord(String payload) {
-        properties = getProperties();
-
         if (!isValid(payload))
             return null;
 
         return extractRecord(payload);
     }
 
-    private Record extractRecord(String payload) {
+    protected boolean isValid(String payload) {
+        return properties != null && parser.isPayloadValid(payload);
+    }
+
+    protected Record extractRecord(String payload) {
         Record record = new Record();
 
         record.setSessionId(getSessionId(payload));
@@ -64,10 +75,11 @@ private Record extractRecord(String payload) {
 
     protected String getValue(String payload, String fieldName) {
         String value = properties.get(fieldName);
-        if(value == null) return null;
+        if (value == null)
+            return null;
 
         // If it is static literal we dont need custom parser
-        if(value.startsWith("{") && value.endsWith("}"))
+        if (value.startsWith("{") && value.endsWith("}"))
             return value.substring(1, value.indexOf("}"));
 
         return parse(payload, value);
@@ -127,10 +139,8 @@ protected Data getData(String payload, String sqlString) {
         }
 
         Data data = new Data();
-        // If it reaches out this point it is a regex parsing and object and verb are
-        // not null
-        String object = getValue(payload, OBJECT);
-        String verb = getValue(payload, VERB);
+        String object = getNotNullString(getValue(payload, OBJECT));
+        String verb = getNotNullString(getValue(payload, VERB));
         Construct construct = new Construct();
         Sentence sentence = new Sentence(verb);
         SentenceObject sentenceObject = new SentenceObject(object);
@@ -143,9 +153,8 @@ protected Data getData(String payload, String sqlString) {
         return data;
     }
 
-    protected Boolean isIpv6(String payload) {
-        String value = getValue(payload, IS_IPV6);
-        return Boolean.parseBoolean(value);
+    protected String getNotNullString(String value) {
+        return value != null ? value : DEFAULT_STRING;
     }
 
     protected Integer getMinDst(String payload) {
@@ -184,13 +193,14 @@ protected String getSqlString(String payload) {
     // the Record. If the timestamp is not available, it sets default values.
     protected Time getTimestamp(String payload) {
         String value = getValue(payload, TIMESTAMP);
-        Time time;
         if (value != null) {
-            time = parseTimestamp(value);
-        } else {
-            time = new Time(0L, 0, 0);
+            try {
+                return parseTimestamp(value);
+            } catch (Exception e) {
+                logger.error("Time {} is invalid.", value, e);
+            }
         }
-        return time;
+        return new Time(0L, 0, 0);
     }
 
     protected SessionLocator getSessionLocator(String payload, String sessionId) {
@@ -203,10 +213,9 @@ protected SessionLocator getSessionLocator(String payload, String sessionId) {
         sessionLocator.setClientIp(DEFAULT_IP);
         sessionLocator.setServerIp(DEFAULT_IP);
 
-        boolean isIpV6 = isIpv6(payload);
         String clientIp = getClientIp(payload);
         String clientIpv6 = getClientIpv6(payload);
-        if (isIpV6 && inetAddressValidator.isValidInet6Address(clientIpv6)) {
+        if (inetAddressValidator.isValidInet6Address(clientIpv6)) {
             // If client IP is IPv6, set both client and server to IPv6
             sessionLocator.setIpv6(true);
             sessionLocator.setClientIpv6(clientIpv6);
@@ -332,7 +341,7 @@ protected String getServerHostName(String payload) {
     protected String getServerType(String payload) {
         // this has been validated before
         if (parseUsingSniffer)
-            return SqlParser.getServerType((String) properties.get(SNIFFER_PARSER));
+            return SqlParser.getServerType(properties.get(SNIFFER_PARSER));
 
         String value = getValue(payload, SERVER_TYPE);
         return value != null ? value : DEFAULT_STRING;
@@ -341,14 +350,14 @@ protected String getServerType(String payload) {
     protected String getLanguage(String payload) {
         // this has been validated before
         if (parseUsingSniffer)
-            return (String) properties.get(SNIFFER_PARSER);
+            return properties.get(SNIFFER_PARSER);
 
         return Accessor.LANGUAGE_FREE_TEXT_STRING;
     }
 
     protected String getDataType(String payload) {
         if (parseUsingSniffer)
-            return (String) properties.get(DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL);
+            return properties.get(DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL);
 
         return DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL;
     }
@@ -376,14 +385,18 @@ protected Integer getServerPort(String sessionId, String payload) {
 
     public abstract String getConfigFilePath();
 
-    public Map<String, String> getProperties() {
+    public Map<String, String> getProperties() throws InvalidConfigurationException {
+        Map<String, String> props;
         try {
             String content = new String(Files.readAllBytes(Paths.get(getConfigFilePath())));
-            return mapper.readValue(content, HashMap.class);
+            props = mapper.readValue(content, new TypeReference<HashMap<String, String>>() {
+            });
+            if (!arePropertiesValid(props))
+                throw new InvalidConfigurationException("The configuration file data is invalid.");
         } catch (IOException e) {
-            logger.error("Error reading properties from config file", e);
-            return null;
+            throw new InvalidConfigurationException("Error reading or parsing the configuration file.");
         }
+        return props;
     }
 
     protected Integer convertToInt(String fieldName, String value) {
@@ -396,27 +409,24 @@ protected Integer convertToInt(String fieldName, String value) {
         return null;
     }
 
-    protected boolean isValid(String payload) {
-        if (properties == null) {
+    protected boolean arePropertiesValid(Map<String, String> props) {
+        if (props == null) {
             logger.error("The provided config file is invalid.");
             return false;
         }
 
-        if (payload == null) {
-            logger.error("The provided payload is null.");
-            return false;
-        }
-
-        hasSqlParsing = SqlParser.hasSqlParsing(properties);
-        parseUsingSniffer = hasSqlParsing && SqlParser.isSnifferParsing(properties);
-        parseUsingRegex = hasSqlParsing && SqlParser.isRegexParsing(properties);
+        hasSqlParsing = SqlParser.hasSqlParsing(props);
+        parseUsingSniffer = hasSqlParsing && SqlParser.isSnifferParsing(props);
+        parseUsingCustomParser = hasSqlParsing && SqlParser.isCustomParsing(props);
 
-        SqlParser.ValidityCase isValid = SqlParser.isValid(properties, hasSqlParsing, parseUsingSniffer, parseUsingRegex);
+        SqlParser.ValidityCase isValid = SqlParser.isValid(props, hasSqlParsing, parseUsingSniffer,
+                parseUsingCustomParser);
         if (!isValid.equals(SqlParser.ValidityCase.VALID)) {
             logger.error(isValid.getDescription());
             return false;
         }
 
         return true;
     }
+
 }

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/ParserFactory.java

@@ -1,23 +1,26 @@
 package com.ibm.guardium.universalconnector.commons.custom_parsing;
 
 import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.IParser;
+import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.JsonParser;
+import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.LeefParser;
 import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.RegexParser;
 
 public class ParserFactory {
-    IParser parser;
-
     public ParserFactory() {
     }
 
     public IParser getParser(ParserType parserType) {
-        //For now we only support Regex
-        //if(parserType.equals(ParserType.regex))
+        if (parserType.equals(ParserType.json))
+            return new JsonParser();
+        else if (parserType.equals(ParserType.leef))
+            return new LeefParser();
+
         return new RegexParser();
     }
 
     public enum ParserType {
         regex,
         json,
-        xml
+        leef
     }
 }

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/SqlParser.java

@@ -5,7 +5,7 @@
 import static com.ibm.guardium.universalconnector.commons.custom_parsing.PropertyConstant.*;
 
 public class SqlParser {
-    static List<String> validParsers = new ArrayList<>(Arrays.asList("REGEX", "SNIFFER", "JAVA"));
+    static List<String> validParsers = new ArrayList<>(Arrays.asList("CUSTOM_PARSER", "SNIFFER", "JAVA"));
     static Map<String, String> validSnifferParsers;
 
     static {
@@ -40,7 +40,8 @@ static String getServerType(String language) {
         return validSnifferParsers.get(language);
     }
 
-    static ValidityCase isValid(Map<String, String> properties, boolean hasSqlParsing, boolean parseUsingSniffer, boolean parseUsingRegex){
+    static ValidityCase isValid(Map<String, String> properties, boolean hasSqlParsing, boolean parseUsingSniffer,
+            boolean parseUsingCustomParser) {
         if (!hasSqlParsing)
             return ValidityCase.VALID;
 
@@ -52,7 +53,7 @@ static ValidityCase isValid(Map<String, String> properties, boolean hasSqlParsin
             String snifferParser = properties.get(SNIFFER_PARSER);
             if (snifferParser == null || !validSnifferParsers.containsKey(snifferParser))
                 return ValidityCase.INVALID_SNIFFER_PARSER;
-        } else if(parseUsingRegex){
+        } else if (parseUsingCustomParser) {
             String object = properties.get(OBJECT);
             if (object == null || object.isEmpty())
                 return ValidityCase.NULL_OBJECT;
@@ -65,23 +66,23 @@ static ValidityCase isValid(Map<String, String> properties, boolean hasSqlParsin
         return ValidityCase.VALID;
     }
 
-    static boolean hasSqlParsing(Map<String, String> properties) {
+    public static boolean hasSqlParsing(Map<String, String> properties) {
         return Boolean.parseBoolean(properties.get(SQL_PARSING_ACTIVE));
     }
 
     static boolean isSnifferParsing(Map<String, String> properties) {
         String parsingType = properties.get(PARSING_TYPE);
-        return parsingType!= null && parsingType.equalsIgnoreCase("SNIFFER");
+        return parsingType != null && parsingType.equalsIgnoreCase("SNIFFER");
     }
 
-    static boolean isRegexParsing(Map<String, String> properties) {
+    static boolean isCustomParsing(Map<String, String> properties) {
         String parsingType = properties.get(PARSING_TYPE);
-        return parsingType!= null && parsingType.equalsIgnoreCase("REGEX");
+        return parsingType != null && parsingType.equalsIgnoreCase("CUSTOM_PARSER");
     }
 
     enum ValidityCase {
         VALID("The SQL Parsing is valid"),
-        INVALID_PARSING_TYPE("Parsing type can only be REGEX or SNIFFER"),
+        INVALID_PARSING_TYPE("Parsing type can only be CUSTOM_PARSER (for Regex and Json) or SNIFFER"),
         INVALID_SNIFFER_PARSER("Sniffer Parser is invalid."),
         NULL_OBJECT("The object field cannot be null."),
         NULL_VERB("The verb field cannot be null.");

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/excepton/InvalidConfigurationException.java

@@ -0,0 +1,7 @@
+package com.ibm.guardium.universalconnector.commons.custom_parsing.excepton;
+
+public class InvalidConfigurationException extends Exception {
+    public InvalidConfigurationException(String msg) {
+        super(msg);
+    }
+}

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/parsers/IParser.java

@@ -2,4 +2,6 @@
 
 public interface IParser {
     String parse(String payload, String key);
+
+    boolean isPayloadValid(String payload);
 }

common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/parsers/JsonParser.java

@@ -0,0 +1,31 @@
+package com.ibm.guardium.universalconnector.commons.custom_parsing.parsers;
+
+import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.json_parser.JsonUtil;
+
+import java.util.Map;
+
+public class JsonParser implements IParser {
+
+    private final JsonUtil util = new JsonUtil();
+    Map<String, String> extractedProperties;
+
+    @Override
+    public String parse(String payload, String key) {
+        if (key == null)
+            return null;
+        return extractedProperties.get(key);
+    }
+
+    @Override
+    public boolean isPayloadValid(String payload) {
+        if (payload == null)
+            return false;
+
+        extractedProperties = parsePayload(payload);
+        return extractedProperties != null && !extractedProperties.isEmpty();
+    }
+
+    public Map<String, String> parsePayload(String payload) {
+        return util.getMap(payload);
+    }
+}