From d8bc17b17009467945dd47c9b90d061bbf6de899 Mon Sep 17 00:00:00 2001 From: blotus Date: Wed, 16 Oct 2024 16:55:32 +0200 Subject: [PATCH] wineventlog: add support for replaying evtx files (#3278) --- go.mod | 2 +- go.sum | 2 + .../modules/wineventlog/test_files/Setup.evtx | Bin 0 -> 69632 bytes .../wineventlog/wineventlog_windows.go | 160 ++++++++++++++++-- ...og_test.go => wineventlog_windows_test.go} | 94 ++++++++-- pkg/exprhelpers/helpers.go | 2 +- pkg/exprhelpers/xml.go | 100 ++++++++--- 7 files changed, 313 insertions(+), 47 deletions(-) create mode 100644 pkg/acquisition/modules/wineventlog/test_files/Setup.evtx rename pkg/acquisition/modules/wineventlog/{wineventlog_test.go => wineventlog_windows_test.go} (73%) diff --git a/go.mod b/go.mod index b02d3b76840..f28f21c6eb4 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/appleboy/gin-jwt/v2 v2.9.2 github.com/aws/aws-lambda-go v1.47.0 github.com/aws/aws-sdk-go v1.52.0 - github.com/beevik/etree v1.3.0 + github.com/beevik/etree v1.4.1 github.com/blackfireio/osinfo v1.0.5 github.com/bluele/gcache v0.0.2 github.com/buger/jsonparser v1.1.1 diff --git a/go.sum b/go.sum index 7aaea1587b8..b2bd77c9915 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/aws/aws-sdk-go v1.52.0 h1:ptgek/4B2v/ljsjYSEvLQ8LTD+SQyrqhOOWvHc/VGPI github.com/aws/aws-sdk-go v1.52.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beevik/etree v1.3.0 h1:hQTc+pylzIKDb23yYprodCWWTt+ojFfUZyzU09a/hmU= github.com/beevik/etree v1.3.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc= +github.com/beevik/etree v1.4.1 h1:PmQJDDYahBGNKDcpdX8uPy1xRCwoCGVUiW669MEirVI= +github.com/beevik/etree v1.4.1/go.mod h1:gPNJNaBGVZ9AwsidazFZyygnd+0pAU38N4D+WemwKNs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/pkg/acquisition/modules/wineventlog/test_files/Setup.evtx b/pkg/acquisition/modules/wineventlog/test_files/Setup.evtx new file mode 100644 index 0000000000000000000000000000000000000000..2c4f8b0f680d0cf86be8d25825fc6a94576a5fe2 GIT binary patch literal 69632 zcmeI43yf6N8OOhwogHQ$%J2&c39?Ah6Pcbrlt@+rTV43oH9TccCb? zZm3#g+G-n3Oi8MtPZMiss|_XAXlrX~2@un?#zNYrG-~=turW3EvHkzg+yl#H=q||Y zEb0GD?%vnA_k8#F{qJ{P7glz6wsdu8ESxS14h&))u_Cj|NI1&}+UMQgSyOb&2fB$a z76B0u0TB=Z5fA|p5CIVo0TB=Z5f~MLmEE1oH}`HqA3vL>2kY?uAdWx2#O$r5W@~pJ zxF^VV&MV&kdvo7>aQF>o#?6jQF#CHk%83m>V-`!8EyXaz*`|XdTyrnVWSWUGnf?*v z-xD0?vE4Y<8M5ap?7tV~S+fs?%IFF1#dA*u+t(s*3+}l;bRXs!o<0)XbNDfCKL;-7 znl8Wm%va8zZ?+Hj_Wr)|jz=1jt6q;(L?Tumxp;Ew32za%7bN%p@Ur)he)WrAX}#?M zJPC6xtFpVX$p>WGUtVl0ZM$Wx*9Nfvc`WhUgOkr!6x$B#v2L8~$BE)dy=_Le0o!U< z+7sqV9FNLidYr@~miQ@ZD z*^5SohF*oQyx;n6jN4kf3wIuHC7MN-4GrB2ySzt_?@U;$D}6i4>OihLakc?Vix=bS z$71#g7~6xBX33;Q;G;Y_y zfpiNIvA78DRBUswOdZchi?%z_0gTBr+Cm`1Uzx8DvFdsEhl41OVPw(v~R!-X)-Ozl8+`!t(3Oh zkBfXj*z-=AEqB>yN4K-YV{JpemX_cRFFjsr%W%g57*4ee;7POhO-`_iID-P+UjMTy zW6{|=@tMwW(L~B+Xs8Ee_#m?Hxe`3U4rll_S5Vfyt7r$jX|$#EHzk2)HzH#{et{JU zOT-yk>Dv376Ck`fVCw-SODs_osdRE;FlPE3B)S|q(-2axvztz{pUxw9$I&E zDaj0g%vOZst6ux&CSMV7JGr|nwd)nTcvt4H|1O6m49D;ajbc5@v|<+}=b^qBq7Pp)o^|jU;ffJlZbI1|?o`ITl>^Xj#-bDP@O0(8;4YV79nW6F zBJaeHFBjh(Muq6ah$WbRVUYinGEU}S3lpe0;74V4Ad>MKmIL?2 zVsJJd5+vw!uAg_Vy2$1X8uYy!)lFN-2W{}f$;zM()k+(Dvkgbn2qPKy-p#h^6IXjd z_9LHHcpD(nqrG6%hwB*R!sRsJZj0gZ)poVjp~|YmuNkE^Xqr?eTH{)$P{mt+thTn{uq}hfkCoQwFd^5sc|H-)ZIvXtX zdkS9P4)4?VP4NB*zIV=s)9L>N)_6x4K`z_XvG7>!`fk>`8Reyk*jn>nZ+k9kKVY z9{#6J8lVCG+MRCF&<0za>~GY#o=FFCHM^}^mzR2?-ZtWj4*a-9buAvPj@TYS!_TEp z&Hwgom3fOtx!R+?#ITMYM?5+Lv%F;m)zSY&dz5jdB+$W2zrE(``O`tJddZg<*1-Ti zXwn0F^iz*xJko(2HCUQ)MGcl( zr@k~4HA)?|A|9bZ>e6HH?%bO{9WBk#PWB~+b@U=S#%o}fx2&K#nl`T5$!FG4e&W$6 z$6Tl*wwJj^+I!cd-$vcxyL%M3H4k9uIX^OMw|~C-d@6>t4nJi+57Ra z)sZhTtfTud2sH(rcyF0>lp`*Mb(F2=IF{1@y=2#Hc{f8DY&m7yMk$N)JXP@@{MD;1N_SH}T=i;kFGv2JHI18O^g*UqBlfvB7Y}|XS=g7}I zT6(H=ex7ta6wST?3ynS6uQXhIRA}Xy`jwyk!N|k>b)QzAf?N9PC$QzmhatwD#Fc z^QWg={Ze0ISWhWTGpL%YD7L)InQcoB(cj=jX_s!0qj&j8%Ut(BC z_hBOFQ7qoFg6e3z#-*{D4?2hT5l7}ir8w6q3`x2&Lg8n1C_ zZ1i*%acNpsM%L21<+A+gC|6waC5Cl$4C5ScVeys~R7V#|T*}Y<;#tNe)^q;aR{_2ZW0m}m4QZrxiPH7-9MEKS(U_Bs^aib)}FSwZzQWn9fO@?T-_y9bUiHM+lx zF*?NW@x!C}9J}WjDrcfoSUAeWu~?29ai9P2kY7UZ>pbSA9+{V;PyCl89Es&O6dQ4T zgy@u=U-aWl{@>&F{ql)Fj;)@2iD5lW$AnNDKI-+BNlz!n^T)BCf@)4H5fA|p5CIVo z0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p z5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo z0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p5CIVo0TB=Z5fA|p a5CIVo0TB=Z5fA|p5CIVo0TCE?1pW&`k>URU literal 0 HcmV?d00001 diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 087c20eb70e..ca40363155b 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -5,7 +5,9 @@ import ( "encoding/xml" "errors" "fmt" + "net/url" "runtime" + "strconv" "strings" "syscall" "time" @@ -30,7 +32,7 @@ type WinEventLogConfiguration struct { EventLevel string `yaml:"event_level"` EventIDs []int `yaml:"event_ids"` XPathQuery string `yaml:"xpath_query"` - EventFile string `yaml:"event_file"` + EventFile string PrettyName string `yaml:"pretty_name"` } @@ -48,10 +50,13 @@ type QueryList struct { } type Select struct { - Path string `xml:"Path,attr"` + Path string `xml:"Path,attr,omitempty"` Query string `xml:",chardata"` } +// 0 identifies the local machine in windows APIs +const localMachine = 0 + var linesRead = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cs_winevtlogsource_hits_total", @@ -212,20 +217,28 @@ func (w *WinEventLogSource) getEvents(out chan types.Event, t *tomb.Tomb) error } } -func (w *WinEventLogSource) generateConfig(query string) (*winlog.SubscribeConfig, error) { +func (w *WinEventLogSource) generateConfig(query string, live bool) (*winlog.SubscribeConfig, error) { var config winlog.SubscribeConfig var err error - // Create a subscription signaler. - config.SignalEvent, err = windows.CreateEvent( - nil, // Default security descriptor. - 1, // Manual reset. - 1, // Initial state is signaled. - nil) // Optional name. - if err != nil { - return &config, fmt.Errorf("windows.CreateEvent failed: %v", err) + if live { + // Create a subscription signaler. + config.SignalEvent, err = windows.CreateEvent( + nil, // Default security descriptor. + 1, // Manual reset. + 1, // Initial state is signaled. + nil) // Optional name. + if err != nil { + return &config, fmt.Errorf("windows.CreateEvent failed: %v", err) + } + config.Flags = wevtapi.EvtSubscribeToFutureEvents + } else { + config.ChannelPath, err = syscall.UTF16PtrFromString(w.config.EventFile) + if err != nil { + return &config, fmt.Errorf("syscall.UTF16PtrFromString failed: %v", err) + } + config.Flags = wevtapi.EvtQueryFilePath | wevtapi.EvtQueryForwardDirection } - config.Flags = wevtapi.EvtSubscribeToFutureEvents config.Query, err = syscall.UTF16PtrFromString(query) if err != nil { return &config, fmt.Errorf("syscall.UTF16PtrFromString failed: %v", err) @@ -283,7 +296,7 @@ func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, Metr return err } - w.evtConfig, err = w.generateConfig(w.query) + w.evtConfig, err = w.generateConfig(w.query, true) if err != nil { return err } @@ -292,6 +305,78 @@ func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, Metr } func (w *WinEventLogSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { + if !strings.HasPrefix(dsn, "wineventlog://") { + return fmt.Errorf("invalid DSN %s for wineventlog source, must start with wineventlog://", dsn) + } + + w.logger = logger + w.config = WinEventLogConfiguration{} + + dsn = strings.TrimPrefix(dsn, "wineventlog://") + + args := strings.Split(dsn, "?") + + if args[0] == "" { + return errors.New("empty wineventlog:// DSN") + } + + if len(args) > 2 { + return errors.New("too many arguments in DSN") + } + + w.config.EventFile = args[0] + + if len(args) == 2 && args[1] != "" { + params, err := url.ParseQuery(args[1]) + if err != nil { + return fmt.Errorf("failed to parse DSN parameters: %w", err) + } + + for key, value := range params { + switch key { + case "log_level": + if len(value) != 1 { + return errors.New("log_level must be a single value") + } + lvl, err := log.ParseLevel(value[0]) + if err != nil { + return fmt.Errorf("failed to parse log_level: %s", err) + } + w.logger.Logger.SetLevel(lvl) + case "event_id": + for _, id := range value { + evtid, err := strconv.Atoi(id) + if err != nil { + return fmt.Errorf("failed to parse event_id: %s", err) + } + w.config.EventIDs = append(w.config.EventIDs, evtid) + } + case "event_level": + if len(value) != 1 { + return errors.New("event_level must be a single value") + } + w.config.EventLevel = value[0] + } + } + } + + var err error + + //FIXME: handle custom xpath query + w.query, err = w.buildXpathQuery() + + if err != nil { + return fmt.Errorf("buildXpathQuery failed: %w", err) + } + + w.logger.Debugf("query: %s\n", w.query) + + w.evtConfig, err = w.generateConfig(w.query, false) + + if err != nil { + return fmt.Errorf("generateConfig failed: %w", err) + } + return nil } @@ -300,10 +385,57 @@ func (w *WinEventLogSource) GetMode() string { } func (w *WinEventLogSource) SupportedModes() []string { - return []string{configuration.TAIL_MODE} + return []string{configuration.TAIL_MODE, configuration.CAT_MODE} } func (w *WinEventLogSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + + handle, err := wevtapi.EvtQuery(localMachine, w.evtConfig.ChannelPath, w.evtConfig.Query, w.evtConfig.Flags) + + if err != nil { + return fmt.Errorf("EvtQuery failed: %v", err) + } + + defer winlog.Close(handle) + + publisherCache := make(map[string]windows.Handle) + defer func() { + for _, h := range publisherCache { + winlog.Close(h) + } + }() + +OUTER_LOOP: + for { + select { + case <-t.Dying(): + w.logger.Infof("wineventlog is dying") + return nil + default: + evts, err := w.getXMLEvents(w.evtConfig, publisherCache, handle, 500) + if err == windows.ERROR_NO_MORE_ITEMS { + log.Info("No more items") + break OUTER_LOOP + } else if err != nil { + return fmt.Errorf("getXMLEvents failed: %v", err) + } + w.logger.Debugf("Got %d events", len(evts)) + for _, evt := range evts { + w.logger.Tracef("Event: %s", evt) + if w.metricsLevel != configuration.METRICS_NONE { + linesRead.With(prometheus.Labels{"source": w.name}).Inc() + } + l := types.Line{} + l.Raw = evt + l.Module = w.GetName() + l.Labels = w.config.Labels + l.Time = time.Now() + l.Src = w.name + l.Process = true + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} + } + } + } return nil } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_test.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go similarity index 73% rename from pkg/acquisition/modules/wineventlog/wineventlog_test.go rename to pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go index ae6cb776909..9afef963669 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_test.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go @@ -4,7 +4,6 @@ package wineventlogacquisition import ( "context" - "runtime" "testing" "time" @@ -19,9 +18,8 @@ import ( ) func TestBadConfiguration(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("Skipping test on non-windows OS") - } + exprhelpers.Init(nil) + tests := []struct { config string expectedErr string @@ -64,9 +62,8 @@ xpath_query: test`, } func TestQueryBuilder(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("Skipping test on non-windows OS") - } + exprhelpers.Init(nil) + tests := []struct { config string expectedQuery string @@ -130,10 +127,8 @@ event_level: bla`, } func TestLiveAcquisition(t *testing.T) { + exprhelpers.Init(nil) ctx := context.Background() - if runtime.GOOS != "windows" { - t.Skip("Skipping test on non-windows OS") - } tests := []struct { config string @@ -227,3 +222,82 @@ event_ids: to.Wait() } } + +func TestOneShotAcquisition(t *testing.T) { + tests := []struct { + name string + dsn string + expectedCount int + expectedErr string + expectedConfigureErr string + }{ + { + name: "non-existing file", + dsn: `wineventlog://foo.evtx`, + expectedCount: 0, + expectedErr: "The system cannot find the file specified.", + }, + { + name: "empty DSN", + dsn: `wineventlog://`, + expectedCount: 0, + expectedConfigureErr: "empty wineventlog:// DSN", + }, + { + name: "existing file", + dsn: `wineventlog://test_files/Setup.evtx`, + expectedCount: 24, + expectedErr: "", + }, + { + name: "filter on event_id", + dsn: `wineventlog://test_files/Setup.evtx?event_id=2`, + expectedCount: 1, + }, + { + name: "filter on event_id", + dsn: `wineventlog://test_files/Setup.evtx?event_id=2&event_id=3`, + expectedCount: 24, + }, + } + + exprhelpers.Init(nil) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + lineCount := 0 + to := &tomb.Tomb{} + c := make(chan types.Event) + f := WinEventLogSource{} + err := f.ConfigureByDSN(test.dsn, map[string]string{"type": "wineventlog"}, log.WithField("type", "windowseventlog"), "") + + if test.expectedConfigureErr != "" { + assert.Contains(t, err.Error(), test.expectedConfigureErr) + return + } + + require.NoError(t, err) + + go func() { + for { + select { + case <-c: + lineCount++ + case <-to.Dying(): + return + } + } + }() + + err = f.OneShotAcquisition(c, to) + if test.expectedErr != "" { + assert.Contains(t, err.Error(), test.expectedErr) + } else { + require.NoError(t, err) + + time.Sleep(2 * time.Second) + assert.Equal(t, test.expectedCount, lineCount) + } + }) + } +} diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 6b7eb0840e9..9bc991a8f2d 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -129,7 +129,7 @@ func Init(databaseClient *database.Client) error { dataFileRegex = make(map[string][]*regexp.Regexp) dataFileRe2 = make(map[string][]*re2.Regexp) dbClient = databaseClient - + XMLCacheInit() return nil } diff --git a/pkg/exprhelpers/xml.go b/pkg/exprhelpers/xml.go index 75758e18316..0b550bdb641 100644 --- a/pkg/exprhelpers/xml.go +++ b/pkg/exprhelpers/xml.go @@ -1,43 +1,103 @@ package exprhelpers import ( + "errors" + "sync" + "time" + "github.com/beevik/etree" + "github.com/bluele/gcache" + "github.com/cespare/xxhash/v2" log "github.com/sirupsen/logrus" ) -var pathCache = make(map[string]etree.Path) +var ( + pathCache = make(map[string]etree.Path) + rwMutex = sync.RWMutex{} + xmlDocumentCache gcache.Cache +) + +func compileOrGetPath(path string) (etree.Path, error) { + rwMutex.RLock() + compiledPath, ok := pathCache[path] + rwMutex.RUnlock() + + if !ok { + var err error + compiledPath, err = etree.CompilePath(path) + if err != nil { + return etree.Path{}, err + } + + rwMutex.Lock() + pathCache[path] = compiledPath + rwMutex.Unlock() + } + + return compiledPath, nil +} + +func getXMLDocumentFromCache(xmlString string) (*etree.Document, error) { + cacheKey := xxhash.Sum64String(xmlString) + cacheObj, err := xmlDocumentCache.Get(cacheKey) + + if err != nil && !errors.Is(err, gcache.KeyNotFoundError) { + return nil, err + } + + doc, ok := cacheObj.(*etree.Document) + if !ok || cacheObj == nil { + doc = etree.NewDocument() + if err := doc.ReadFromString(xmlString); err != nil { + return nil, err + } + if err := xmlDocumentCache.Set(cacheKey, doc); err != nil { + log.Warnf("Could not set XML document in cache: %s", err) + } + } + + return doc, nil +} + +func XMLCacheInit() { + gc := gcache.New(50) + // Short cache expiration because we each line we read is different, but we can call multiple times XML helpers on each of them + gc.Expiration(5 * time.Second) + gc = gc.LRU() + + xmlDocumentCache = gc.Build() +} // func XMLGetAttributeValue(xmlString string, path string, attributeName string) string { func XMLGetAttributeValue(params ...any) (any, error) { xmlString := params[0].(string) path := params[1].(string) attributeName := params[2].(string) - if _, ok := pathCache[path]; !ok { - compiledPath, err := etree.CompilePath(path) - if err != nil { - log.Errorf("Could not compile path %s: %s", path, err) - return "", nil - } - pathCache[path] = compiledPath + + compiledPath, err := compileOrGetPath(path) + if err != nil { + log.Errorf("Could not compile path %s: %s", path, err) + return "", nil } - compiledPath := pathCache[path] - doc := etree.NewDocument() - err := doc.ReadFromString(xmlString) + doc, err := getXMLDocumentFromCache(xmlString) if err != nil { log.Tracef("Could not parse XML: %s", err) return "", nil } + elem := doc.FindElementPath(compiledPath) if elem == nil { log.Debugf("Could not find element %s", path) return "", nil } + attr := elem.SelectAttr(attributeName) if attr == nil { log.Debugf("Could not find attribute %s", attributeName) return "", nil } + return attr.Value, nil } @@ -45,26 +105,24 @@ func XMLGetAttributeValue(params ...any) (any, error) { func XMLGetNodeValue(params ...any) (any, error) { xmlString := params[0].(string) path := params[1].(string) - if _, ok := pathCache[path]; !ok { - compiledPath, err := etree.CompilePath(path) - if err != nil { - log.Errorf("Could not compile path %s: %s", path, err) - return "", nil - } - pathCache[path] = compiledPath + + compiledPath, err := compileOrGetPath(path) + if err != nil { + log.Errorf("Could not compile path %s: %s", path, err) + return "", nil } - compiledPath := pathCache[path] - doc := etree.NewDocument() - err := doc.ReadFromString(xmlString) + doc, err := getXMLDocumentFromCache(xmlString) if err != nil { log.Tracef("Could not parse XML: %s", err) return "", nil } + elem := doc.FindElementPath(compiledPath) if elem == nil { log.Debugf("Could not find element %s", path) return "", nil } + return elem.Text(), nil }