diff --git a/build.gradle b/build.gradle index 70b9e0bd3..c81cc9dc0 100644 --- a/build.gradle +++ b/build.gradle @@ -155,7 +155,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}" implementation "org.antlr:antlr4-runtime:4.10.1" implementation "com.cronutils:cron-utils:9.1.6" - api "org.opensearch:common-utils:${common_utils_version}@jar" + api files("/Users/snistala/Documents/opensearch/common-utils/build/libs/common-utils-3.0.0.0-SNAPSHOT.jar") api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index fe1402e59..bec6ef8ae 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -10,6 +10,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -660,6 +661,13 @@ public void getRuleFieldMappings(String logType, ActionListener getIocFieldsList(String logType) { + LogType logTypeByName = builtinLogTypeLoader.getLogTypeByName(logType); + if(logTypeByName == null) + return Collections.emptyList(); + return logTypeByName.getIocFieldsList(); + } + public void getRuleFieldMappingsAllSchemas(String logType, ActionListener> listener) { if (builtinLogTypeLoader.logTypeExists(logType)) { diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java index 7acc0d1f3..8cee7ab23 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/LogType.java +++ b/src/main/java/org/opensearch/securityanalytics/model/LogType.java @@ -6,14 +6,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; public class LogType implements Writeable { @@ -25,12 +24,16 @@ public class LogType implements Writeable { private static final String RAW_FIELD = "raw_field"; public static final String ECS = "ecs"; public static final String OCSF = "ocsf"; + public static final String IOC_FIELDS = "ioc_fields"; + public static final String IOC = "ioc"; + public static final String FIELDS = "fields"; private String id; private String name; private String description; private Boolean isBuiltIn; private List mappings; + private List iocFieldsList; public LogType(StreamInput sin) throws IOException { this.id = sin.readString(); @@ -38,14 +41,16 @@ public LogType(StreamInput sin) throws IOException { this.name = sin.readString(); this.description = sin.readString(); this.mappings = sin.readList(Mapping::readFrom); + this.iocFieldsList = sin.readList(IocFields::readFrom); } - public LogType(String id, String name, String description, boolean isBuiltIn, List mappings) { + public LogType(String id, String name, String description, boolean isBuiltIn, List mappings, List iocFieldsList) { this.id = id; this.name = name; this.description = description; this.isBuiltIn = isBuiltIn; this.mappings = mappings == null ? List.of() : mappings; + this.iocFieldsList = iocFieldsList == null ? List.of() : iocFieldsList; } public LogType(Map logTypeAsMap) { @@ -62,6 +67,17 @@ public LogType(Map logTypeAsMap) { new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) ).collect(Collectors.toList()); } + if(logTypeAsMap.containsKey(IOC_FIELDS)) { + List> iocFieldsList = (List>) logTypeAsMap.get(IOC_FIELDS); + if (iocFieldsList.size() > 0) { + this.iocFieldsList = new ArrayList<>(mappings.size()); + this.iocFieldsList = iocFieldsList.stream().map(e -> + new IocFields(e.get(IOC).toString(), (List) e.get(FIELDS)) + ).collect(Collectors.toList()); + } + } else { + iocFieldsList = Collections.emptyList(); + } } public String getName() { @@ -74,6 +90,10 @@ public String getDescription() { public boolean getIsBuiltIn() { return isBuiltIn; } + public List getIocFieldsList() { + return iocFieldsList; + } + public List getMappings() { return mappings; } @@ -85,6 +105,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(description); out.writeCollection(mappings); + out.writeCollection(iocFieldsList); } @Override @@ -134,4 +155,43 @@ public static Mapping readFrom(StreamInput sin) throws IOException { } } + /** + * stores information of list of field names that contain information for given IoC (Indicator of Compromise). + */ + public static class IocFields implements Writeable { + private final String ioc; + + private final List fields; + + public IocFields(String ioc, List fields) { + this.ioc = ioc; + this.fields = fields; + } + + public IocFields(StreamInput sin) throws IOException { + this.ioc = sin.readString(); + this.fields = sin.readStringList(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(ioc); + out.writeStringCollection(fields); + } + + public String getIoc() { + return ioc; + } + + public List getFields() { + return fields; + } + + + public static IocFields readFrom(StreamInput sin) throws IOException { + return new IocFields(sin); + } + } + + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java index b0891f413..fb4bb744e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java @@ -1,15 +1,19 @@ package org.opensearch.securityanalytics.threatIntel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.DocLevelQuery; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -19,33 +23,56 @@ public class DetectorThreatIntelService { + private static final Logger log = LogManager.getLogger(DetectorThreatIntelService.class); + private final ThreatIntelFeedDataService threatIntelFeedDataService; public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) { this.threatIntelFeedDataService = threatIntelFeedDataService; } + /** * Convert the feed data IOCs into query string query format to create doc level queries. */ - public DocLevelQuery createDocLevelQueryFromThreatIntelList( - List tifdList, String docLevelQueryId + public List createDocLevelQueriesFromThreatIntelList( + List iocFieldList, List tifdList, Detector detector ) { + List queries = new ArrayList<>(); Set iocs = tifdList.stream().map(ThreatIntelFeedData::getIocValue).collect(Collectors.toSet()); - String query = buildQueryStringQueryWithIocList(iocs); - return new DocLevelQuery( - docLevelQueryId, tifdList.get(0).getFeedId(), - Collections.singletonList("*"), - query, - Collections.singletonList("threat_intel") - ); + //ioc types supported by log type + List logTypeIocs = iocFieldList.stream().map(LogType.IocFields::getIoc).collect(Collectors.toList()); + // filter out ioc types not supported for given log types + Map> iocTypeToValues = tifdList.stream().filter(t -> logTypeIocs.contains(t.getIocType())) + .collect(Collectors.groupingBy( + ThreatIntelFeedData::getIocType, + Collectors.mapping(ThreatIntelFeedData::getIocValue, Collectors.toSet()) + )); + + for (Map.Entry> entry : iocTypeToValues.entrySet()) { + String query = buildQueryStringQueryWithIocList(iocs); + List fields = iocFieldList.stream().filter(t -> entry.getKey().matches(t.getIoc())).findFirst().get().getFields(); + + // create doc + for (String field : fields) { //todo increase max clause count from 1024 + queries.add(new DocLevelQuery( + constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(), + Collections.emptyList(), + String.format(query, field), + List.of("threat_intel", entry.getKey() /*ioc_type*/) + )); + } + } + return queries; } private String buildQueryStringQueryWithIocList(Set iocs) { StringBuilder sb = new StringBuilder(); + sb.append("%s"); + sb.append(":"); sb.append("("); for (String ioc : iocs) { - if (sb.length() > 2) { + if (sb.length() > 4) { sb.append(" OR "); } sb.append(ioc); @@ -55,30 +82,30 @@ private String buildQueryStringQueryWithIocList(Set iocs) { return sb.toString(); } - public void createDocLevelQueryFromThreatIntel(Detector detector, ActionListener listener) { + public void createDocLevelQueryFromThreatIntel(List iocFieldList, Detector detector, ActionListener> listener) { try { - if (detector.getThreatIntelEnabled() == false) { - listener.onResponse(null); + if (false == detector.getThreatIntelEnabled() || iocFieldList.isEmpty()) { + listener.onResponse(Collections.emptyList()); return; - } + CountDownLatch latch = new CountDownLatch(1); threatIntelFeedDataService.getThreatIntelFeedData(new ActionListener<>() { @Override public void onResponse(List threatIntelFeedData) { if (threatIntelFeedData.isEmpty()) { - listener.onResponse(null); + listener.onResponse(Collections.emptyList()); } else { - listener.onResponse(createDocLevelQueryFromThreatIntelList( - threatIntelFeedData, - detector.getName() + "_threat_intel" + UUID.randomUUID() - )); + listener.onResponse( + createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector) + ); } latch.countDown(); } @Override public void onFailure(Exception e) { + log.error("Failed to get threat intel feeds for doc level query creation", e); listener.onFailure(e); latch.countDown(); } @@ -86,11 +113,16 @@ public void onFailure(Exception e) { latch.await(30, TimeUnit.SECONDS); } catch (InterruptedException e) { + log.error("Failed to create doc level queries from threat intel feeds", e); listener.onFailure(e); } } + private static String constructId(Detector detector, String iocType) { + return detector.getName() + "_threat_intel_" + iocType + "_" + UUID.randomUUID(); + } + public void updateDetectorsWithLatestThreatIntelRules() { } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 4805179df..3eb0a5112 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -88,6 +88,7 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Value; import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; @@ -323,7 +324,9 @@ private void createMonitorFromQueries(List> rulesById, Detect monitorResponses.add(addedFirstMonitorResponse); saveWorkflow(rulesById, detector, monitorResponses, refreshPolicy, listener); }, - listener::onFailure + e -> { + listener.onFailure(e); + } ); } } @@ -653,30 +656,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, Collections.emptyList(), actualQuery, tags); docLevelQueries.add(docLevelQuery); } - try { - if (detector.getThreatIntelEnabled()) { - CountDownLatch countDownLatch = new CountDownLatch(1); - detectorThreatIntelService.createDocLevelQueryFromThreatIntel(detector, new ActionListener<>() { - @Override - public void onResponse(DocLevelQuery dlq) { - if (dlq != null) - docLevelQueries.add(dlq); - countDownLatch.countDown(); - } - - @Override - public void onFailure(Exception e) { - // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data - log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); - countDownLatch.countDown(); - } - }); - countDownLatch.await(); - } - } catch (Exception e) { - // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data - log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); - } + addThreatIntelBasedDocLevelQueries(detector, docLevelQueries); DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); docLevelMonitorInputs.add(docLevelMonitorInput); @@ -707,6 +687,39 @@ public void onFailure(Exception e) { return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } + private void addThreatIntelBasedDocLevelQueries(Detector detector, List docLevelQueries) { + try { + + if (detector.getThreatIntelEnabled()) { + List iocFieldsList = logTypeService.getIocFieldsList(detector.getDetectorType()); + if (iocFieldsList == null || iocFieldsList.isEmpty()) { + + } else { + CountDownLatch countDownLatch = new CountDownLatch(1); + detectorThreatIntelService.createDocLevelQueryFromThreatIntel(iocFieldsList, detector, new ActionListener<>() { + @Override + public void onResponse(List dlqs) { + if (dlqs != null) + docLevelQueries.addAll(dlqs); + countDownLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data + log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); + countDownLatch.countDown(); + } + }); + countDownLatch.await(); + } + } + } catch (Exception e) { + // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data + log.error("Failed to convert threat intel feed to doc level query. Proceeding with detector creation", e); + } + } + /** * Creates doc level monitor which generates per document alerts for the findings of the bucket level delegate monitors in a workflow. * This monitor has match all query applied to generate the alerts per each finding doc. diff --git a/src/main/resources/OSMapping/test_windows_logtype.json b/src/main/resources/OSMapping/test_windows_logtype.json index 7491a954c..816cba666 100644 --- a/src/main/resources/OSMapping/test_windows_logtype.json +++ b/src/main/resources/OSMapping/test_windows_logtype.json @@ -2,6 +2,12 @@ "name": "test_windows", "description": "Test Log Type used by tests. It is created as a lightweight log type for integration tests", "is_builtin": true, + "ioc_fields": [ + { + "ioc": "ip", + "fields": ["windows-hostname"] + } + ], "mappings": [ { "raw_field":"EventID", diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index abc9caad8..65417ed39 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1373,6 +1373,46 @@ public static String randomDoc(int severity, int version, String opCode) { } + //Add IPs in HostName field. + public static String randomDocWithIpIoc(int severity, int version, String ioc) { + String doc = "{\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"%s\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":%s,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":%s,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"blahblah\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + return String.format(Locale.ROOT, doc, ioc, severity, version); + + } + public static String randomDoc() { return "{\n" + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 15e9f9bad..f46fd0efb 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -39,6 +39,7 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntel; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; @@ -1088,6 +1089,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + assertEquals(2, response.getHits().getTotalHits().value); assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -1118,7 +1120,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt List iocs = getThreatIntelFeedIocs(3); int i=1; for (String ioc : iocs) { - indexDoc(index, i+"", randomDoc(5, 3, ioc)); + indexDoc(index, i+"", randomDocWithIpIoc(5, 3, ioc)); i++; } String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); @@ -1133,7 +1135,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt assertEquals(2, noOfSigmaRuleMatches); String threatIntelDocLevelQueryId = docLevelQueryResults.keySet().stream().filter(id -> id.contains(detector.getName() + "_threat_intel")).findAny().get(); ArrayList docs = (ArrayList) docLevelQueryResults.get(threatIntelDocLevelQueryId); - assertEquals(docs.size(), 2); + assertEquals(docs.size(), 3); // // Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(detector)); //