Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change ruleId if it exists #628

Merged
merged 2 commits into from
Oct 4, 2023
Merged

Change ruleId if it exists #628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e4aeac9
Ingest others_cloud category first
lezzago Oct 3, 2023
2bcacee
Add integ test
lezzago Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,10 +272,16 @@
private void ingestQueries(Map<String, List<String>> logIndexToRules, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener<BulkResponse> listener) throws SigmaError, IOException {
List<Rule> queries = new ArrayList<>();

for (Map.Entry<String, List<String>> logIndexToRule: logIndexToRules.entrySet()) {
Map<String, String> fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(logIndexToRule.getKey());
// Moving others_cloud to the top so those queries are indexed first and can be overwritten if other categories
// contain the same rules. Tracking issue: https://github.com/opensearch-project/security-analytics/issues/630
List<String> categories = new ArrayList<>(logIndexToRules.keySet());

Check warning on line 277 in src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java#L277 

Added line #L277 was not covered by tests
if (categories.remove("others_cloud")) {
categories.add(0, "others_cloud");

Check warning on line 279 in src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java#L279 

Added line #L279 was not covered by tests
}
for (String category: categories) {
Map<String, String> fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(category);

Check warning on line 282 in src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java#L282 

Added line #L282 was not covered by tests
final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true);
queries.addAll(getQueries(backend, logIndexToRule.getKey(), logIndexToRule.getValue()));
queries.addAll(getQueries(backend, category, logIndexToRules.get(category)));

Check warning on line 284 in src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java#L284 

Added line #L284 was not covered by tests
}
loadRules(queries, refreshPolicy, indexTimeout, listener, true);
}
Expand Down
56 changes: 56 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,17 @@
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase;
import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;
import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.model.DetectorRule;
import org.opensearch.securityanalytics.model.Rule;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -192,6 +197,57 @@ public void testSearchingPrepackagedRules() throws IOException {
Assert.assertEquals(5, ((Map<String, Object>) ((Map<String, Object>) responseBody.get("hits")).get("total")).get("value"));
}

public void testSearchingForDuplicatedPrepackagedRules() throws IOException {
String gworkspaceRequest = "{\n" +
" \"query\": {\n" +
" \"nested\": {\n" +
" \"path\": \"rule\",\n" +
" \"query\": {\n" +
" \"bool\": {\n" +
" \"must\": [\n" +
" { \"match\": {\"rule.category\": \"gworkspace\"}}\n" +
" ]\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}";

Response gworkSpaceSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"),
new StringEntity(gworkspaceRequest), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(gworkSpaceSearchResponse));

String azureRequest = "{\n" +
" \"query\": {\n" +
" \"nested\": {\n" +
" \"path\": \"rule\",\n" +
" \"query\": {\n" +
" \"bool\": {\n" +
" \"must\": [\n" +
" { \"match\": {\"rule.category\": \"azure\"}}\n" +
" ]\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}";

Response azureSearchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"),
new StringEntity(azureRequest), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(azureSearchResponse));

ClassLoader classLoader = getClass().getClassLoader();
int gworkspaceFileCount = new File(classLoader.getResource("rules/gworkspace").getFile()).listFiles().length;
int azureFileCount = new File(classLoader.getResource("rules/azure").getFile()).listFiles().length;

// Verify azure and gworkspace categories have the right number of rules even though they
// conflict with others_cloud category
Map<String, Object> gworkspaceResponseBody = asMap(gworkSpaceSearchResponse);
Assert.assertEquals(gworkspaceFileCount, ((Map<String, Object>) ((Map<String, Object>) gworkspaceResponseBody.get("hits")).get("total")).get("value"));
Map<String, Object> azureResponseBody = asMap(azureSearchResponse);
Assert.assertEquals(azureFileCount, ((Map<String, Object>) ((Map<String, Object>) azureResponseBody.get("hits")).get("total")).get("value"));
}

@SuppressWarnings("unchecked")
public void testSearchingPrepackagedRulesByMitreAttackID() throws IOException {
String request = "{\n" +
Expand Down
Loading