Merge pull request #145 from Security-Onion-Solutions/dev

Remove sortby clause when querying related case events (bug in ES?)
Security-Onion-Solutions · Oct 25, 2022 · 600743d · 600743d
2 parents 2eea44f + 88e6851
commit 600743d
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 2 deletions.
diff --git a/server/modules/elastic/elasticcasestore.go b/server/modules/elastic/elasticcasestore.go
@@ -18,6 +18,7 @@ import (
   "github.com/security-onion-solutions/securityonion-soc/server"
   "github.com/security-onion-solutions/securityonion-soc/web"
   "regexp"
+  "sort"
   "strconv"
   "time"
 )
@@ -524,13 +525,30 @@ func (store *ElasticCasestore) GetRelatedEvents(ctx context.Context, caseId stri
   err = store.validateId(caseId, "caseId")
   if err == nil {
     events = make([]*model.RelatedEvent, 0)
-    query := fmt.Sprintf(`_index:"%s" AND %skind:"related" AND %srelated.caseId:"%s" | sortby %srelated.fields.timestamp^`, store.index, store.schemaPrefix, store.schemaPrefix, caseId, store.schemaPrefix)
+    // JBE 10/20/2022: Remove sortby due to issue with Elastic 8.4 causing incompatible sort field types
+    //  | sortby %srelated.fields.timestamp^
+    query := fmt.Sprintf(`_index:"%s" AND %skind:"related" AND %srelated.caseId:"%s"`, store.index, store.schemaPrefix, store.schemaPrefix, caseId)
     var objects []interface{}
     objects, err = store.getAll(ctx, query, store.maxAssociations)
     if err == nil {
       for _, obj := range objects {
         events = append(events, obj.(*model.RelatedEvent))
       }
+
+      // JBE 10/20/2022: Manually sort the related events by the timestamp field, in ascending order. This can remain
+      // in place even if the above ES issue is resolved.
+      sort.Slice(events, func(a, b int) bool {
+        if ts_a, ts_a_exists := events[a].Fields["timestamp"]; ts_a_exists {
+          if ts_a_typed, ts_a_correct_type := ts_a.(time.Time); ts_a_correct_type {
+            if ts_b, ts_b_exists := events[b].Fields["timestamp"]; ts_b_exists {
+              if ts_b_typed, ts_b_correct_type := ts_b.(time.Time); ts_b_correct_type {
+                return ts_a_typed.Before(ts_b_typed)
+              }
+            }
+          }
+        }
+        return false
+      })
     }
   }
   return events, err

diff --git a/server/modules/elastic/elasticcasestore_test.go b/server/modules/elastic/elasticcasestore_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/security-onion-solutions/securityonion-soc/web"
 	"github.com/stretchr/testify/assert"
 	"testing"
+	"time"
 )
 
 func TestInit(tester *testing.T) {
@@ -966,18 +967,47 @@ func TestGetRelatedEvents(tester *testing.T) {
 	fakeEventStore := server.NewFakeEventstore()
 	store.server.Eventstore = fakeEventStore
 	ctx := context.WithValue(context.Background(), web.ContextKeyRequestorId, "myRequestorId")
-	query := `_index:"myIndex" AND so_kind:"related" AND so_related.caseId:"myCaseId" | sortby so_related.fields.timestamp^`
+	// JBE: 10/20/2022 - Remove sortby and perform it manually due to ES issue with flattened fields
+	//query := `_index:"myIndex" AND so_kind:"related" AND so_related.caseId:"myCaseId" | sortby so_related.fields.timestamp^`
+	query := `_index:"myIndex" AND so_kind:"related" AND so_related.caseId:"myCaseId"`
 	eventPayload := make(map[string]interface{})
 	eventPayload["so_kind"] = "related"
 	elasticEvent := &model.EventRecord{
 		Payload: eventPayload,
 	}
 	fakeEventStore.SearchResults[0].Events = append(fakeEventStore.SearchResults[0].Events, elasticEvent)
+
+	// Add a related event with a timestamp field
+	timeA, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+	eventPayloadWithTimestamp := make(map[string]interface{})
+	eventPayloadWithTimestamp["so_kind"] = "related"
+	eventPayloadWithTimestamp["so_related.fields.timestamp"] = timeA
+	elasticEventWithTimestamp := &model.EventRecord{
+		Payload: eventPayloadWithTimestamp,
+	}
+	fakeEventStore.SearchResults[0].Events = append(fakeEventStore.SearchResults[0].Events, elasticEventWithTimestamp)
+
+	// Add another related event with an earlier timestamp field
+	timeB, _ := time.Parse(time.RFC3339, "2006-01-01T15:04:05Z")
+	eventPayloadWithTimestamp2 := make(map[string]interface{})
+	eventPayloadWithTimestamp2["so_kind"] = "related"
+	eventPayloadWithTimestamp2["so_related.fields.timestamp"] = timeB
+	elasticEventWithTimestamp2 := &model.EventRecord{
+		Payload: eventPayloadWithTimestamp2,
+	}
+	fakeEventStore.SearchResults[0].Events = append(fakeEventStore.SearchResults[0].Events, elasticEventWithTimestamp2)
+
 	obj, err := store.GetRelatedEvents(ctx, "myCaseId")
 	assert.NoError(tester, err)
 	assert.Len(tester, fakeEventStore.InputSearchCriterias, 1)
 	assert.Equal(tester, query, fakeEventStore.InputSearchCriterias[0].RawQuery)
 	assert.NotNil(tester, obj)
+
+	// Ensure manual sorting functions as expected (only has effect if the sortby claused is removed)
+	assert.Len(tester, obj, 3)
+	assert.Nil(tester, obj[0].Fields["timestamp"])
+	assert.Equal(tester, timeB, obj[1].Fields["timestamp"])
+	assert.Equal(tester, timeA, obj[2].Fields["timestamp"])
 }
 
 func TestDeleteRelatedEvent(tester *testing.T) {