From e4aeac90fc64d567fb75e32f332179b0e513bdef Mon Sep 17 00:00:00 2001 From: Ashish Agrawal Date: Tue, 3 Oct 2023 14:18:41 -0700 Subject: [PATCH 1/2] Ingest others_cloud category first Signed-off-by: Ashish Agrawal --- .../securityanalytics/util/RuleIndices.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index 53c0a516f..b3018a157 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -272,10 +272,16 @@ private String getRuleCategory(Path folderPath) { private void ingestQueries(Map> logIndexToRules, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener listener) throws SigmaError, IOException { List queries = new ArrayList<>(); - for (Map.Entry> logIndexToRule: logIndexToRules.entrySet()) { - Map fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(logIndexToRule.getKey()); + // Moving others_cloud to the top so those queries are indexed first and can be overwritten + // if other categories contain the same rules + List categories = new ArrayList<>(logIndexToRules.keySet()); + if (categories.remove("others_cloud")) { + categories.add(0, "others_cloud"); + } + for (String category: categories) { + Map fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(category); final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); - queries.addAll(getQueries(backend, logIndexToRule.getKey(), logIndexToRule.getValue())); + queries.addAll(getQueries(backend, category, logIndexToRules.get(category))); } loadRules(queries, refreshPolicy, indexTimeout, listener, true); } From 2bcaceef9f1f8daca0966e665b0be46cff665d81 Mon Sep 17 00:00:00 2001 From: Ashish Agrawal Date: Tue, 3 Oct 2023 16:35:21 -0700 Subject: [PATCH 2/2] Add integ test Signed-off-by: Ashish Agrawal --- .../securityanalytics/util/RuleIndices.java | 4 +- .../resthandler/RuleRestApiIT.java | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index b3018a157..e3df6020f 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -272,8 +272,8 @@ private String getRuleCategory(Path folderPath) { private void ingestQueries(Map> logIndexToRules, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener listener) throws SigmaError, IOException { List queries = new ArrayList<>(); - // Moving others_cloud to the top so those queries are indexed first and can be overwritten - // if other categories contain the same rules + // Moving others_cloud to the top so those queries are indexed first and can be overwritten if other categories + // contain the same rules. Tracking issue: https://github.com/opensearch-project/security-analytics/issues/630 List categories = new ArrayList<>(logIndexToRules.keySet()); if (categories.remove("others_cloud")) { categories.add(0, "others_cloud"); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 9a483b9e4..6628ae7fd 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -22,12 +22,17 @@ import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.Rule; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -192,6 +197,57 @@ public void testSearchingPrepackagedRules() throws IOException { Assert.assertEquals(5, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); } + public void testSearchingForDuplicatedPrepackagedRules() throws IOException { + String gworkspaceRequest = "{\n" + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"gworkspace\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Response gworkSpaceSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"), + new StringEntity(gworkspaceRequest), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(gworkSpaceSearchResponse)); + + String azureRequest = "{\n" + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"azure\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Response azureSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"), + new StringEntity(azureRequest), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(azureSearchResponse)); + + ClassLoader classLoader = getClass().getClassLoader(); + int gworkspaceFileCount = new File(classLoader.getResource("rules/gworkspace").getFile()).listFiles().length; + int azureFileCount = new File(classLoader.getResource("rules/azure").getFile()).listFiles().length; + + // Verify azure and gworkspace categories have the right number of rules even though they + // conflict with others_cloud category + Map gworkspaceResponseBody = asMap(gworkSpaceSearchResponse); + Assert.assertEquals(gworkspaceFileCount, ((Map) ((Map) gworkspaceResponseBody.get("hits")).get("total")).get("value")); + Map azureResponseBody = asMap(azureSearchResponse); + Assert.assertEquals(azureFileCount, ((Map) ((Map) azureResponseBody.get("hits")).get("total")).get("value")); + } + @SuppressWarnings("unchecked") public void testSearchingPrepackagedRulesByMitreAttackID() throws IOException { String request = "{\n" +