diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 78801d4a00e..f0ad6e6a5c1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Auditbeat* +- Add linux capabilities to processes in the system/process. {pull}37453[37453] +- Add opt-in eBPF backend for file_integrity module. {pull}37223[37223] +- Add process data to file events (Linux only, eBPF backend). {pull}38199[38199] *Filebeat* diff --git a/NOTICE.txt b/NOTICE.txt index c4ae3f617b5..eb8588c7783 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12287,11 +12287,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/ebpfevents -Version: v0.4.0 +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.4.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.5.0/LICENSE.txt: The https://github.com/elastic/ebpfevents repository contains source code under various licenses: @@ -22921,6 +22921,45 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/tklauser/go-sysconf +Version: v0.3.10 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: + +BSD 3-Clause License + +Copyright (c) 2018-2021, Tobias Klauser +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/tsg/go-daemon Version: v0.0.0-20200207173439-e704b93fd89b @@ -36661,11 +36700,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/cilium/ebpf -Version: v0.12.3 +Version: v0.13.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.12.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.13.2/LICENSE: MIT License @@ -38697,11 +38736,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.14.5 +Version: v1.14.3 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.3/LICENSE: MIT License @@ -39304,6 +39343,37 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/go-quicktest/qt +Version: v1.101.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-quicktest/qt@v1.101.0/LICENSE: + +MIT License + +Copyright (c) 2017 Canonical Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-sourcemap/sourcemap Version: v2.1.2+incompatible @@ -49663,27 +49733,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/pkg/diff -Version: v0.0.0-20210226163009-20ebb0f2a09e -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/pkg/diff@v0.0.0-20210226163009-20ebb0f2a09e/LICENSE: - -Copyright 2018 Joshua Bleecher Snyder - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/pmezard/go-difflib Version: v1.0.0 @@ -49967,11 +50016,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golan -------------------------------------------------------------------------------- Dependency : github.com/rogpeppe/go-internal -Version: v1.9.0 +Version: v1.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.11.0/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. @@ -50873,45 +50922,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/tklauser/go-sysconf -Version: v0.3.10 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: - -BSD 3-Clause License - -Copyright (c) 2018-2021, Tobias Klauser -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/tklauser/numcpus Version: v0.4.0 diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index 22813a47f22..b282aaaf3d2 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -134,6 +134,7 @@ type Event struct { Action Action `json:"action"` // Action (like created, updated). Hashes map[HashType]Digest `json:"hash,omitempty"` // File hashes. ParserResults mapstr.M `json:"file,omitempty"` // Results from running file parsers. + Process *Process `json:"process,omitempty"` // Process data. Available only on Linux when using the eBPF backend. // Metadata rtt time.Duration // Time taken to collect the info. @@ -141,6 +142,33 @@ type Event struct { hashFailed bool // Set when hashing the file failed. } +// Process contain information about a process. +// These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +type Process struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + // Process name. Sometimes called program name or similar. + Name string `json:"name,omitempty"` + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + // Process id. + PID uint32 `json:"pid,omitempty"` +} + // Metadata contains file metadata. type Metadata struct { Inode uint64 `json:"inode"` @@ -354,6 +382,24 @@ func buildMetricbeatEvent(e *Event, existedBefore bool) mb.Event { } } + if e.Process != nil { + process := mapstr.M{ + "pid": e.Process.PID, + "name": e.Process.Name, + "entity_id": e.Process.EntityID, + "user": mapstr.M{ + "id": e.Process.User.ID, + "name": e.Process.User.Name, + }, + "group": mapstr.M{ + "id": e.Process.Group.ID, + "name": e.Process.Group.Name, + }, + } + + out.MetricSetFields.Put("process", process) + } + if len(e.Hashes) > 0 { hashes := make(mapstr.M, len(e.Hashes)) for hashType, digest := range e.Hashes { diff --git a/auditbeat/module/file_integrity/event_linux.go b/auditbeat/module/file_integrity/event_linux.go index 7643d03a6b4..c0eb2d57b15 100644 --- a/auditbeat/module/file_integrity/event_linux.go +++ b/auditbeat/module/file_integrity/event_linux.go @@ -26,6 +26,7 @@ import ( "strconv" "time" + "github.com/elastic/beats/v7/libbeat/ebpf/sys" "github.com/elastic/ebpfevents" ) @@ -41,7 +42,9 @@ func NewEventFromEbpfEvent( path, target string action Action metadata Metadata + process Process err error + errors []error ) switch ee.Type { case ebpfevents.EventTypeFileCreate: @@ -54,7 +57,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileCreateEvent.SymlinkTargetPath + metadata, err = metadataFromFileCreate(fileCreateEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileCreate(fileCreateEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileRename: action = Moved @@ -65,7 +77,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileRenameEvent.SymlinkTargetPath + metadata, err = metadataFromFileRename(fileRenameEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileRename(fileRenameEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileDelete: action = Deleted @@ -76,6 +97,11 @@ func NewEventFromEbpfEvent( return event, false } target = fileDeleteEvent.SymlinkTargetPath + + process, err = processFromFileDelete(fileDeleteEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileModify: fileModifyEvent := ee.Body.(*ebpfevents.FileModify) @@ -92,7 +118,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileModifyEvent.SymlinkTargetPath + metadata, err = metadataFromFileModify(fileModifyEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileModify(fileModifyEvent) + if err != nil { + errors = append(errors, err) + } } event := Event{ @@ -102,10 +137,8 @@ func NewEventFromEbpfEvent( Info: &metadata, Source: SourceEBPF, Action: action, - errors: make([]error, 0), - } - if err != nil { - event.errors = append(event.errors, err) + Process: &process, + errors: errors, } if event.Action == Deleted { @@ -115,7 +148,6 @@ func NewEventFromEbpfEvent( case FileType: fillHashes(&event, path, maxFileSize, hashTypes, fileParsers) case SymlinkType: - var err error event.TargetPath, err = filepath.EvalSymlinks(event.Path) if err != nil { event.errors = append(event.errors, err) @@ -147,6 +179,59 @@ func metadataFromFileModify(evt *ebpfevents.FileModify) (Metadata, error) { return md, err } +func newProcess(pid uint32, start uint64, comm string, euid, egid uint32) (Process, error) { + var ( + p Process + err error + ) + + t, err := sys.TimeFromNsSinceBoot(start) + if err != nil { + return p, err + } + + p.EntityID, err = sys.EntityID(pid, t) + if err != nil { + return p, err + } + p.Name = comm + p.PID = pid + + p.User.ID = strconv.FormatUint(uint64(euid), 10) + u, err := user.LookupId(p.User.ID) + if err == nil { + p.User.Name = u.Username + } else { + p.User.Name = "n/a" + } + + p.Group.ID = strconv.FormatUint(uint64(egid), 10) + g, err := user.LookupGroupId(p.Group.ID) + if err == nil { + p.Group.Name = g.Name + } else { + p.Group.Name = "n/a" + } + + return p, nil +} + +func processFromFileCreate(evt *ebpfevents.FileCreate) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileRename(evt *ebpfevents.FileRename) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileModify(evt *ebpfevents.FileModify) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileDelete(evt *ebpfevents.FileDelete) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + func fillFileInfo(md *Metadata, finfo ebpfevents.FileInfo) error { md.Inode = finfo.Inode md.UID = finfo.Uid diff --git a/auditbeat/module/file_integrity/event_linux_test.go b/auditbeat/module/file_integrity/event_linux_test.go index 1a440afb8f1..beac9878909 100644 --- a/auditbeat/module/file_integrity/event_linux_test.go +++ b/auditbeat/module/file_integrity/event_linux_test.go @@ -21,7 +21,6 @@ package file_integrity import ( "os" - "os/user" "testing" "github.com/stretchr/testify/assert" @@ -40,13 +39,25 @@ func TestNewEventFromEbpfEvent(t *testing.T) { Inode: 1234, Mode: os.FileMode(0o644), Size: 2345, - Uid: 3456, - Gid: 4567, + Uid: uint32(os.Geteuid()), + Gid: uint32(os.Getegid()), }, Path: "/foo", SymlinkTargetPath: "/bar", + Creds: ebpfevents.CredInfo{ + Ruid: 1, + Rgid: 2, + Euid: uint32(os.Geteuid()), + Egid: uint32(os.Getegid()), + Suid: 5, + Sgid: 6, + }, }, } + event, ok := NewEventFromEbpfEvent( + ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) + assert.True(t, ok) + expectedEvent := Event{ Action: Created, Path: "/foo", @@ -54,21 +65,22 @@ func TestNewEventFromEbpfEvent(t *testing.T) { Info: &Metadata{ Type: FileType, Inode: 1234, - UID: 3456, - GID: 4567, + UID: uint32(os.Geteuid()), + GID: uint32(os.Getegid()), Size: 2345, - Owner: "n/a", - Group: "n/a", + Owner: event.Info.Owner, + Group: event.Info.Group, Mode: os.FileMode(0o644), }, - Source: SourceEBPF, - errors: []error{user.UnknownUserIdError(3456)}, + Process: event.Process, // 1:1 copy this as it changes on every machine + Source: SourceEBPF, + errors: nil, } - - event, ok := NewEventFromEbpfEvent( - ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) - assert.True(t, ok) event.Timestamp = expectedEvent.Timestamp assert.Equal(t, expectedEvent, event) + assert.NotEqual(t, "", event.Process.EntityID) + assert.NotEqual(t, 0, event.Process.PID) + assert.NotEqual(t, 0, event.Process.User.ID) + assert.NotEqual(t, "", event.Process.User.Name) } diff --git a/auditbeat/tests/system/test_file_integrity.py b/auditbeat/tests/system/test_file_integrity.py index e6b03306c3a..64062dd0a77 100644 --- a/auditbeat/tests/system/test_file_integrity.py +++ b/auditbeat/tests/system/test_file_integrity.py @@ -5,6 +5,10 @@ from auditbeat import * +if platform.platform().split('-')[0] == 'Linux': + import pwd + + def is_root(): if 'geteuid' not in dir(os): return False @@ -100,6 +104,16 @@ def wait_startup(self, backend, dir): # may differ self.wait_log_contains(escape_path(dir), max_timeout=30, ignore_case=True) + def _assert_process_data(self, event, backend): + if backend != "ebpf": + return + assert event["process.entity_id"] != "" + assert event["process.executable"] == "pytest" + assert event["process.pid"] == os.getpid() + assert int(event["process.user.id"]) == os.geteuid() + assert event["process.user.name"] == pwd.getpwuid(os.geteuid()).pw_name + assert int(event["process.group.id"]) == os.getegid() + def _test_non_recursive(self, backend): """ file_integrity monitors watched directories (non recursive). @@ -172,6 +186,8 @@ def _test_non_recursive(self, backend): # assert file inside subdir is not reported assert self.log_contains(file3) is False + self._assert_process_data(objs[0], backend) + @unittest.skipIf(os.getenv("CI") is not None and platform.system() == 'Darwin', 'Flaky test: https://github.com/elastic/beats/issues/24678') def test_non_recursive__fsnotify(self): @@ -252,6 +268,8 @@ def _test_recursive(self, backend): file_events(objs, file1, ['created']) file_events(objs, file2, ['created']) + self._assert_process_data(objs[0], backend) + def test_recursive__fsnotify(self): self._test_recursive("fsnotify") diff --git a/go.mod b/go.mod index 745d47673b6..9e8c91fc742 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.21 +go 1.21.8 require ( cloud.google.com/go/bigquery v1.55.0 @@ -201,7 +201,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.4.0 + github.com/elastic/ebpfevents v0.5.0 github.com/elastic/elastic-agent-autodiscover v0.6.8 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 @@ -224,6 +224,7 @@ require ( github.com/pkg/xattr v0.4.9 github.com/sergi/go-diff v1.3.1 github.com/shirou/gopsutil/v3 v3.22.10 + github.com/tklauser/go-sysconf v0.3.10 go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.8 go.elastic.co/apm/module/apmhttp/v2 v2.4.8 go.elastic.co/apm/v2 v2.4.8 @@ -271,7 +272,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cilium/ebpf v0.12.3 // indirect + github.com/cilium/ebpf v0.13.2 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -360,7 +361,6 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index ff0abc92363..d839df6a98e 100644 --- a/go.sum +++ b/go.sum @@ -442,8 +442,8 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= -github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= +github.com/cilium/ebpf v0.13.2 h1:uhLimLX+jF9BTPPvoCUYh/mBeoONkjgaJ9w9fn0mRj4= +github.com/cilium/ebpf v0.13.2/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -671,8 +671,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI= -github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.5.0 h1:QkyMAYWo3fXFbYtXAXU8sZu2SQ4LXVYC6gLXIWXy02E= +github.com/elastic/ebpfevents v0.5.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU= github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8= github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM= @@ -772,8 +772,8 @@ github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -919,6 +919,8 @@ github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -1639,7 +1641,6 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1732,8 +1733,9 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rootless-containers/rootlesskit v1.1.0 h1:cRaRIYxY8oce4eE/zeAUZhgKu/4tU1p9YHN4+suwV7M= github.com/rootless-containers/rootlesskit v1.1.0/go.mod h1:H+o9ndNe7tS91WqU0/+vpvc+VaCd7TCIWaJjnV0ujUo= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= diff --git a/libbeat/ebpf/sys/sys.go b/libbeat/ebpf/sys/sys.go new file mode 100644 index 00000000000..4156321a321 --- /dev/null +++ b/libbeat/ebpf/sys/sys.go @@ -0,0 +1,68 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package sys + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "sync" + "time" + + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" +) + +var ( + hostInfoOnce = sync.OnceValues(func() (types.HostInfo, error) { + host, err := sysinfo.Host() + + if host == nil { + return types.HostInfo{}, err + } + + return host.Info(), err + }) +) + +// EntityID creates an ID that uniquely identifies this process across machines. +func EntityID(pid uint32, start time.Time) (string, error) { + info, err := hostInfoOnce() + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write([]byte(info.UniqueID)); err != nil { + return "", err + } + if err := binary.Write(h, binary.LittleEndian, int64(pid)); err != nil { + return "", err + } + if err := binary.Write(h, binary.LittleEndian, int64(start.Nanosecond())); err != nil { + return "", err + } + + sum := h.Sum(nil) + if len(sum) > 12 { + sum = sum[:12] + } + return base64.RawStdEncoding.EncodeToString(sum), nil +} diff --git a/libbeat/ebpf/sys/time.go b/libbeat/ebpf/sys/time.go new file mode 100644 index 00000000000..7dca6454c32 --- /dev/null +++ b/libbeat/ebpf/sys/time.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package sys + +import ( + "sync" + "time" + + "github.com/tklauser/go-sysconf" +) + +var ( + ticksPerSecondOnce = sync.OnceValues(func() (uint64, error) { + ticks, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) + if err != nil { + return 0, err + } + + return uint64(ticks), err + }) +) + +func TicksToNs(ticks uint64) (uint64, error) { + tps, err := ticksPerSecondOnce() + if err != nil { + return 0, err + } + + return ticks * uint64(time.Second.Nanoseconds()) / tps, nil +} + +func TimeFromNsSinceBoot(ns uint64) (time.Time, error) { + info, err := hostInfoOnce() + if err != nil { + return time.Time{}, err + } + + reduced, err := reduceTimestampPrecision(ns) + if err != nil { + return time.Time{}, err + } + + return info.BootTime.Add(time.Duration(reduced)), nil +} + +// When generating an `entity_id` in ECS we need to reduce the precision of a +// process's start time to that of procfs. Process start times can come from either +// eBPF (high precision) or other sources. We must reduce them all to the +// lowest common denominator such that entity ID's generated are always consistent. +func reduceTimestampPrecision(ns uint64) (uint64, error) { + tps, err := ticksPerSecondOnce() + if err != nil { + return 0, err + } + + return ns - (ns % (uint64(time.Second.Nanoseconds()) / tps)), nil +} diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index fc4227738c3..87133cda11d 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,3 +1,7 @@ +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 @@ -13,8 +17,6 @@ cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -58,7 +60,6 @@ pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 redis==4.4.4 -requests==2.31.0 semver==2.8.1 six==1.15.0 stomp.py==4.1.22 @@ -67,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index fc4227738c3..87133cda11d 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -1,3 +1,7 @@ +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 @@ -13,8 +17,6 @@ cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -58,7 +60,6 @@ pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 redis==4.4.4 -requests==2.31.0 semver==2.8.1 six==1.15.0 stomp.py==4.1.22 @@ -67,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index b6da0417316..a16b942032d 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,5 @@ FROM golang:1.21.8 +COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ apt update \ @@ -10,6 +11,8 @@ RUN \ python3-venv \ libaio-dev \ unzip \ + libssl-dev \ + libffi-dev \ && rm -rf /var/lib/apt/lists/* # Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts @@ -19,9 +22,12 @@ RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip3 install --upgrade pip==20.1.1 -RUN pip3 install --upgrade docker-compose==1.23.2 RUN pip3 install --upgrade setuptools==47.3.2 RUN pip3 install --upgrade PyYAML==5.3.1 +RUN pip3 install requests==2.31.0 +RUN pip3 install urllib3==1.26.18 +RUN pip3 install docker==6.1.3 +RUN pip3 install git+https://github.com/pkoutsovasilis/compose@v1_fix # Oracle instant client RUN cd /usr/lib \