Appsec: properly populate event (#2943)

cmd/crowdsec-cli/alerts.go

@@ -493,6 +493,7 @@ func (cli *cliAlerts) NewInspectCmd() *cobra.Command {
 				switch cfg.Cscli.Output {
 				case "human":
 					if err := cli.displayOneAlert(alert, details); err != nil {
+						log.Warnf("unable to display alert with id %s: %s", alertID, err)
 						continue
 					}
 				case "json":

cmd/crowdsec/crowdsec.go

@@ -19,6 +19,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/appsec"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -32,6 +33,13 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
 		return nil, nil, fmt.Errorf("while loading context: %w", err)
 	}
 
+	err = exprhelpers.GeoIPInit(hub.GetDataDir())
+
+	if err != nil {
+		//GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
+		log.Warnf("unable to initialize GeoIP: %s", err)
+	}
+
 	// Start loading configs
 	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {

cmd/crowdsec/serve.go

@@ -177,6 +177,9 @@ func ShutdownCrowdsecRoutines() error {
 	// He's dead, Jim.
 	crowdsecTomb.Kill(nil)
 
+	// close the potential geoips reader we have to avoid leaking ressources on reload
+	exprhelpers.GeoIPClose()
+
 	return reterr
 }
 

pkg/acquisition/modules/appsec/utils.go

@@ -1,20 +1,46 @@
 package appsecacquisition
 
 import (
-	"encoding/json"
 	"fmt"
+	"net"
+	"slices"
+	"strconv"
 	"time"
 
 	"github.com/crowdsecurity/coraza/v3/collection"
 	"github.com/crowdsecurity/coraza/v3/types/variables"
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/go-cs-lib/ptr"
+	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 )
 
+var appsecMetaKeys = []string{
+	"id",
+	"name",
+	"method",
+	"uri",
+	"matched_zones",
+	"msg",
+}
+
+func appendMeta(meta models.Meta, key string, value string) models.Meta {
+	if value == "" {
+		return meta
+	}
+
+	meta = append(meta, &models.MetaItems0{
+		Key:   key,
+		Value: value,
+	})
+	return meta
+}
+
 func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
 	//if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI
 	if !inEvt.Appsec.HasInBandMatches {
@@ -23,48 +49,127 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
 	evt := types.Event{}
 	evt.Type = types.APPSEC
 	evt.Process = true
+	sourceIP := inEvt.Parsed["source_ip"]
 	source := models.Source{
-		Value: ptr.Of(inEvt.Parsed["source_ip"]),
-		IP:    inEvt.Parsed["source_ip"],
+		Value: &sourceIP,
+		IP:    sourceIP,
 		Scope: ptr.Of(types.Ip),
 	}
 
+	asndata, err := exprhelpers.GeoIPASNEnrich(sourceIP)
+
+	if err != nil {
+		log.Errorf("Unable to enrich ip '%s' for ASN: %s", sourceIP, err)
+	} else if asndata != nil {
+		record := asndata.(*geoip2.ASN)
+		source.AsName = record.AutonomousSystemOrganization
+		source.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber)
+	}
+
+	cityData, err := exprhelpers.GeoIPEnrich(sourceIP)
+	if err != nil {
+		log.Errorf("Unable to enrich ip '%s' for geo data: %s", sourceIP, err)
+	} else if cityData != nil {
+		record := cityData.(*geoip2.City)
+		source.Cn = record.Country.IsoCode
+		source.Latitude = float32(record.Location.Latitude)
+		source.Longitude = float32(record.Location.Longitude)
+	}
+
+	rangeData, err := exprhelpers.GeoIPRangeEnrich(sourceIP)
+	if err != nil {
+		log.Errorf("Unable to enrich ip '%s' for range: %s", sourceIP, err)
+	} else if rangeData != nil {
+		record := rangeData.(*net.IPNet)
+		source.Range = record.String()
+	}
+
 	evt.Overflow.Sources = make(map[string]models.Source)
-	evt.Overflow.Sources["ip"] = source
+	evt.Overflow.Sources[sourceIP] = source
 
 	alert := models.Alert{}
 	alert.Capacity = ptr.Of(int32(1))
-	alert.Events = make([]*models.Event, 0)
-	alert.Meta = make(models.Meta, 0)
-	for _, key := range []string{"target_uri", "method"} {
+	alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs()))
 
-		valueByte, err := json.Marshal([]string{inEvt.Parsed[key]})
-		if err != nil {
-			log.Debugf("unable to serialize key %s", key)
+	now := ptr.Of(time.Now().UTC().Format(time.RFC3339))
+
+	tmpAppsecContext := make(map[string][]string)
+
+	for _, matched_rule := range inEvt.Appsec.MatchedRules {
+		evtRule := models.Event{}
+
+		evtRule.Timestamp = now
+
+		evtRule.Meta = make(models.Meta, 0)
+
+		for _, key := range appsecMetaKeys {
+
+			if tmpAppsecContext[key] == nil {
+				tmpAppsecContext[key] = make([]string, 0)
+			}
+
+			switch value := matched_rule[key].(type) {
+			case string:
+				evtRule.Meta = appendMeta(evtRule.Meta, key, value)
+				if value != "" && !slices.Contains(tmpAppsecContext[key], value) {
+					tmpAppsecContext[key] = append(tmpAppsecContext[key], value)
+				}
+			case int:
+				val := strconv.Itoa(value)
+				evtRule.Meta = appendMeta(evtRule.Meta, key, val)
+				if val != "" && !slices.Contains(tmpAppsecContext[key], val) {
+					tmpAppsecContext[key] = append(tmpAppsecContext[key], val)
+				}
+			case []string:
+				for _, v := range value {
+					evtRule.Meta = appendMeta(evtRule.Meta, key, v)
+					if v != "" && !slices.Contains(tmpAppsecContext[key], v) {
+						tmpAppsecContext[key] = append(tmpAppsecContext[key], v)
+					}
+				}
+			case []int:
+				for _, v := range value {
+					val := strconv.Itoa(v)
+					evtRule.Meta = appendMeta(evtRule.Meta, key, val)
+					if val != "" && !slices.Contains(tmpAppsecContext[key], val) {
+						tmpAppsecContext[key] = append(tmpAppsecContext[key], val)
+					}
+
+				}
+			default:
+				val := fmt.Sprintf("%v", value)
+				evtRule.Meta = appendMeta(evtRule.Meta, key, val)
+				if val != "" && !slices.Contains(tmpAppsecContext[key], val) {
+					tmpAppsecContext[key] = append(tmpAppsecContext[key], val)
+				}
+
+			}
+		}
+		alert.Events = append(alert.Events, &evtRule)
+	}
+
+	metas := make([]*models.MetaItems0, 0)
+
+	for key, values := range tmpAppsecContext {
+		if len(values) == 0 {
 			continue
 		}
 
+		valueStr, err := alertcontext.TruncateContext(values, alertcontext.MaxContextValueLen)
+		if err != nil {
+			log.Warningf(err.Error())
+		}
+
 		meta := models.MetaItems0{
 			Key:   key,
-			Value: string(valueByte),
-		}
-		alert.Meta = append(alert.Meta, &meta)
-	}
-	matchedZones := inEvt.Appsec.GetMatchedZones()
-	if matchedZones != nil {
-		valueByte, err := json.Marshal(matchedZones)
-		if err != nil {
-			log.Debugf("unable to serialize key matched_zones")
-		} else {
-			meta := models.MetaItems0{
-				Key:   "matched_zones",
-				Value: string(valueByte),
-			}
-			alert.Meta = append(alert.Meta, &meta)
+			Value: valueStr,
 		}
+		metas = append(metas, &meta)
 	}
 
-	alert.EventsCount = ptr.Of(int32(1))
+	alert.Meta = metas
+
+	alert.EventsCount = ptr.Of(int32(len(alert.Events)))
 	alert.Leakspeed = ptr.Of("")
 	alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName())
 	alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash())
@@ -200,7 +305,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR
 	})
 
 	for _, rule := range req.Tx.MatchedRules() {
-		if rule.Message() == "" || rule.DisruptiveAction() == "pass" || rule.DisruptiveAction() == "allow" {
+		if rule.Message() == "" {
 			r.logger.Tracef("discarding rule %d (action: %s)", rule.Rule().ID(), rule.DisruptiveAction())
 			continue
 		}
@@ -242,7 +347,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR
 
 		corazaRule := map[string]interface{}{
 			"id":            rule.Rule().ID(),
-			"uri":           evt.Parsed["uri"],
+			"uri":           evt.Parsed["target_uri"],
 			"rule_type":     kind,
 			"method":        evt.Parsed["method"],
 			"disruptive":    rule.Disruptive(),

pkg/alertcontext/alertcontext.go

@@ -16,7 +16,7 @@ import (
 )
 
 const (
-	maxContextValueLen = 4000
+	MaxContextValueLen = 4000
 )
 
 var alertContext = Context{}
@@ -46,13 +46,13 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
 	}
 
 	if valueLength == 0 {
-		clog.Debugf("No console context value length provided, using default: %d", maxContextValueLen)
-		valueLength = maxContextValueLen
+		clog.Debugf("No console context value length provided, using default: %d", MaxContextValueLen)
+		valueLength = MaxContextValueLen
 	}
 
-	if valueLength > maxContextValueLen {
-		clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, maxContextValueLen)
-		valueLength = maxContextValueLen
+	if valueLength > MaxContextValueLen {
+		clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, MaxContextValueLen)
+		valueLength = MaxContextValueLen
 	}
 
 	alertContext = Context{
@@ -85,7 +85,7 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
 	return nil
 }
 
-func truncate(values []string, contextValueLen int) (string, error) {
+func TruncateContext(values []string, contextValueLen int) (string, error) {
 	valueByte, err := json.Marshal(values)
 	if err != nil {
 		return "", fmt.Errorf("unable to dump metas: %w", err)
@@ -159,7 +159,7 @@ func EventToContext(events []types.Event) (models.Meta, []error) {
 			continue
 		}
 
-		valueStr, err := truncate(values, alertContext.ContextValueLen)
+		valueStr, err := TruncateContext(values, alertContext.ContextValueLen)
 		if err != nil {
 			log.Warningf(err.Error())
 		}

pkg/exprhelpers/expr_lib.go

@@ -1,9 +1,11 @@
 package exprhelpers
 
 import (
+	"net"
 	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cticlient"
+	"github.com/oschwald/geoip2-golang"
 )
 
 type exprCustomFunc struct {
@@ -469,6 +471,27 @@ var exprFuncs = []exprCustomFunc{
 			new(func(string) bool),
 		},
 	},
+	{
+		name:     "GeoIPEnrich",
+		function: GeoIPEnrich,
+		signature: []interface{}{
+			new(func(string) *geoip2.City),
+		},
+	},
+	{
+		name:     "GeoIPASNEnrich",
+		function: GeoIPASNEnrich,
+		signature: []interface{}{
+			new(func(string) *geoip2.ASN),
+		},
+	},
+	{
+		name:     "GeoIPRangeEnrich",
+		function: GeoIPRangeEnrich,
+		signature: []interface{}{
+			new(func(string) *net.IPNet),
+		},
+	},
 }
 
 //go 1.20 "CutPrefix":              strings.CutPrefix,