Skip to content

Commit

Permalink
2.9 backports (#663)
Browse files Browse the repository at this point in the history 
* Change ruleId if it exists

Signed-off-by: Ashish Agrawal <[email protected]>

* Fixes detectorType incompatibility with detector rules (#524)

Signed-off-by: jowg-amazon <[email protected]>
  • Loading branch information
@sbcd90
sbcd90 authored Oct 13, 2023
1 parent 72c46eb commit a941222
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 25 deletions.
4 changes: 2 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,9 +288,9 @@ public void onFailure(Exception e) {
);
}, listener::onFailure);
} else {
// Do nothing if detector doesn't have any monitor
// Failure if detector doesn't have any monitor
if (monitorRequests.isEmpty()) {
listener.onResponse(Collections.emptyList());
listener.onFailure(new OpenSearchStatusException("Detector cannot be created as no compatible rules were provided", RestStatus.BAD_REQUEST));
return;
}

Expand Down
12 changes: 9 additions & 3 deletions src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,16 @@ private String getRuleCategory(Path folderPath) {
private void ingestQueries(Map<String, List<String>> logIndexToRules, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener<BulkResponse> listener) throws SigmaError, IOException {
List<Rule> queries = new ArrayList<>();

for (Map.Entry<String, List<String>> logIndexToRule: logIndexToRules.entrySet()) {
Map<String, String> fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(logIndexToRule.getKey());
// Moving others_cloud to the top so those queries are indexed first and can be overwritten if other categories
// contain the same rules. Tracking issue: https://github.com/opensearch-project/security-analytics/issues/630
List<String> categories = new ArrayList<>(logIndexToRules.keySet());
if (categories.remove("others_cloud")) {
categories.add(0, "others_cloud");
}
for (String category: categories) {
Map<String, String> fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(category);
final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true);
queries.addAll(getQueries(backend, logIndexToRule.getKey(), logIndexToRule.getValue()));
queries.addAll(getQueries(backend, category, logIndexToRules.get(category)));
}
loadRules(queries, refreshPolicy, indexTimeout, listener, true);
}
Expand Down
51 changes: 31 additions & 20 deletions src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,29 +322,40 @@ public void testCreateDetectorWithoutRules() throws IOException {

Detector detector = randomDetector(Collections.emptyList());

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
fail("create detector call should have failed");
} catch (ResponseException ex) {
Assert.assertEquals(400, ex.getResponse().getStatusLine().getStatusCode());
assertTrue(ex.getMessage().contains("Detector cannot be created as no compatible rules were provided"));
}
}

Map<String, Object> responseBody = asMap(createResponse);
public void testCreateDetectorWithIncompatibleDetectorType() throws IOException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

// Verify rules
String request = "{\n" +
" \"query\" : {\n" +
" \"match_all\":{\n" +
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
Assert.assertEquals(0, response.getHits().getTotalHits().value);
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + index + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

String createdId = responseBody.get("_id").toString();
int createdVersion = Integer.parseInt(responseBody.get("_version").toString());
Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId);
Assert.assertTrue("incorrect version", createdVersion > 0);
Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location"));
Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("rule_topic_index"));
Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("findings_index"));
Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("alert_index"));
Response createMappingResponse = client().performRequest(createMappingRequest);
assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());

Detector detector = randomDetector(getPrePackagedRules("ad_ldap"));

try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
fail("create detector call should have failed");
} catch (ResponseException ex) {
Assert.assertEquals(400, ex.getResponse().getStatusLine().getStatusCode());
assertTrue(ex.getMessage().contains("Detector cannot be created as no compatible rules were provided"));
}
}

public void testCreateDetectorWithInvalidCategory() throws IOException {
Expand Down
56 changes: 56 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,18 @@
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase;
import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;
import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.model.DetectorRule;
import org.opensearch.securityanalytics.model.Rule;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -198,6 +203,57 @@ public void testSearchingPrepackagedRules() throws IOException {
Assert.assertEquals(5, ((Map<String, Object>) ((Map<String, Object>) responseBody.get("hits")).get("total")).get("value"));
}

public void testSearchingForDuplicatedPrepackagedRules() throws IOException {
String gworkspaceRequest = "{\n" +
" \"query\": {\n" +
" \"nested\": {\n" +
" \"path\": \"rule\",\n" +
" \"query\": {\n" +
" \"bool\": {\n" +
" \"must\": [\n" +
" { \"match\": {\"rule.category\": \"gworkspace\"}}\n" +
" ]\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}";

Response gworkSpaceSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"),
new StringEntity(gworkspaceRequest), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(gworkSpaceSearchResponse));

String azureRequest = "{\n" +
" \"query\": {\n" +
" \"nested\": {\n" +
" \"path\": \"rule\",\n" +
" \"query\": {\n" +
" \"bool\": {\n" +
" \"must\": [\n" +
" { \"match\": {\"rule.category\": \"azure\"}}\n" +
" ]\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}";

Response azureSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"),
new StringEntity(azureRequest), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(azureSearchResponse));

ClassLoader classLoader = getClass().getClassLoader();
int gworkspaceFileCount = new File(classLoader.getResource("rules/gworkspace").getFile()).listFiles().length;
int azureFileCount = new File(classLoader.getResource("rules/azure").getFile()).listFiles().length;

// Verify azure and gworkspace categories have the right number of rules even though they
// conflict with others_cloud category
Map<String, Object> gworkspaceResponseBody = asMap(gworkSpaceSearchResponse);
Assert.assertEquals(gworkspaceFileCount, ((Map<String, Object>) ((Map<String, Object>) gworkspaceResponseBody.get("hits")).get("total")).get("value"));
Map<String, Object> azureResponseBody = asMap(azureSearchResponse);
Assert.assertEquals(azureFileCount, ((Map<String, Object>) ((Map<String, Object>) azureResponseBody.get("hits")).get("total")).get("value"));
}

@SuppressWarnings("unchecked")
public void testSearchingPrepackagedRulesByMitreAttackID() throws IOException {
String request = "{\n" +
Expand Down

0 comments on commit a941222

Please sign in to comment.