From 3b5d832656dbfc6907dab9be0d19642139f2343a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geyslan=20Greg=C3=B3rio?= Date: Mon, 11 Nov 2024 12:56:55 -0300 Subject: [PATCH] chore(changelog): reduce mem footprint The current Changelog structure consumes a significant amount of memory due to the allocation of metadata for each field. As the number of fields increases, the memory usage grows linearly. Approximately 240 bytes per field were observed just for metadata, excluding the actual data and pointers for each field. To reduce memory consumption, the new changelog.Entries implementation uses a flat slice of any type instead of storing metadata for each field separately. --- | Caches | GOGC | Branch | Heap Use | Heap Growth | Delta | Proctree | | | | | MB (Avg) | | | | |--------|------|------- |----------|-------------|---------|----------| | - | - | main | 28 | - | - | off | | 16384 | - | main | 199 | 181.00 | - | on | | 32768 | - | main | 331 | 313.00 | - | on | | - | 5 | main | 18 | - | - | off | | 16384 | 5 | main | 125 | 107.00 | - | on | | 32768 | 5 | main | 209 | 191.00 | - | on | | - | - | new | 28 | - | - | off | | 16384 | - | new | 119 | 91.00 | -50.28% | on | | 32768 | - | new | 172 | 144.00 | -46.01% | on | | - | 5 | new | 18 | - | - | off | | 16384 | 5 | new | 76 | 58.00 | -54.21% | on | | 32768 | 5 | new | 111 | 93.00 | -48.69% | on | With GOGC set to 5, the new implementation reduces average heap usage by approximately 54% when using cache sizes of 16,384. For cache sizes of 32,768, the reduction is around 48%. When GOGC is set to the default value, the reductions are roughly 50% and 46% for cache sizes of 16,384 and 32,768, respectively. The "Heap in Use" column serves as a good indicator of memory consumption and can assist in determining optimal cache sizes. --- Proctree Stressor Benchmark CPU time (5 threads, 700000 iterations each) | Method | New | New (CPU %) | Old | Old (CPU %) | Overhead | |--------|----------|-------------|-----------|-------------|----------| | Set | 0.41 min | 0.74% | < 0.01 hs | 0.87% | -15.0% | | Get | 0.04 min | 0.07% | < 0.01 hs | 0.87% | -92.0% | --- pkg/changelog/changelog.go | 379 ++++++++------- pkg/changelog/changelog_benchmark_test.go | 85 +--- pkg/changelog/changelog_test.go | 551 +++++++++------------- pkg/proctree/fileinfo.go | 127 +++-- pkg/proctree/process.go | 19 +- pkg/proctree/taskinfo.go | 149 ++++-- 6 files changed, 622 insertions(+), 688 deletions(-) diff --git a/pkg/changelog/changelog.go b/pkg/changelog/changelog.go index 6aef4c269112..1f4d79eeaabc 100644 --- a/pkg/changelog/changelog.go +++ b/pkg/changelog/changelog.go @@ -1,224 +1,267 @@ package changelog import ( + "reflect" "time" - - "github.com/aquasecurity/tracee/pkg/logger" ) -type comparable interface { - ~int | ~float64 | ~string -} - -type item[T comparable] struct { - timestamp time.Time // timestamp of the change - value T // value of the change -} +// Entries -// The changelog package provides a changelog data structure. It is a list of changes, each with a -// timestamp. The changelog can be queried for the value at a given time. +// MemberKind represents the unique identifier for each kind of entry in the changelog. +// It is used to categorize different kinds of changes tracked by the changelog. +type MemberKind uint8 -// ATTENTION: You should use Changelog within a struct and provide methods to access it, -// coordinating access through your struct mutexes. DO NOT EXPOSE the changelog object directly to -// the outside world as it is not thread-safe. +// EntryType defines the data type of the entry value (e.g., Int, Float, String). +// It is also used to determine the type of default value when no entry is found for a given kind. +type EntryType uint8 -type Changelog[T comparable] struct { - changes []item[T] // list of changes - timestamps map[time.Time]struct{} // set of timestamps (used to avoid duplicates) - maxSize int // maximum amount of changes to keep track of +// EntryFlags contains metadata about each `MemberKind`, such as the type of data it stores +// and the maximum number of entries to keep for that kind. +type EntryFlags struct { + Type EntryType // Type of the entry value (e.g., Int, String). + MaxEntries uint8 // Maximum number of entries to store for this kind. } -// NewChangelog creates a new changelog. -func NewChangelog[T comparable](maxSize int) *Changelog[T] { - return &Changelog[T]{ - changes: []item[T]{}, - timestamps: map[time.Time]struct{}{}, - maxSize: maxSize, - } -} +// EntryFlagsMap is a mapping from `MemberKind` to `EntryFlags`, used to configure the changelog behavior. +type EntryFlagsMap map[MemberKind]EntryFlags + +const ( + Byte EntryType = iota + Bool + Int8 + Int16 + Int32 + Int64 + Int + Uint8 + Uint16 + Uint32 + Uint64 + Uint + Float32 + Float64 + String +) -// Getters +// entry is an internal structure representing a single change in the changelog. +// It includes the kind of the entry, the timestamp of the change, and the value of the change. +type entry struct { + k MemberKind // Kind of the member, used to categorize the entry. + t time.Time // Timestamp of when the change occurred. + value any // Value of the change. +} -// GetCurrent: Observation on single element changelog. -// -// If there's one element in the changelog, after the loop, left would be set to 1 if the single -// timestamp is before the targetTime, and 0 if it's equal or after. -// -// BEFORE: If the single timestamp is before the targetTime, when we return -// clv.changes[left-1].value, returns clv.changes[0].value, which is the expected behavior. -// -// AFTER: If the single timestamp is equal to, or after the targetTime, the current logic would -// return a "zero" value because of the condition if left == 0. -// -// We need to find the last change that occurred before or exactly at the targetTime. The binary -// search loop finds the position where a new entry with the targetTime timestamp would be inserted -// to maintain chronological order: -// -// This position is stored in "left". -// -// So, to get the last entry that occurred before the targetTime, we need to access the previous -// position, which is left-1. +// Entries is the main structure that manages a list of changes (entries). +// It keeps track of the configured flags for each kind of entry and the list of recorded entries. // -// GetCurrent returns the latest value of the changelog. -func (clv *Changelog[T]) GetCurrent() T { - if len(clv.changes) == 0 { - return returnZero[T]() - } - - return clv.changes[len(clv.changes)-1].value +// ATTENTION: You should use Entries within a struct and provide methods to access it, +// coordinating access through your struct mutexes. DO NOT EXPOSE the changelog object directly to +// the outside world as it is not thread-safe. +type Entries struct { + entryFlagsMap EntryFlagsMap // Configuration map defining flags for each member kind. + entries []entry // List of recorded entries. } -// Get returns the value of the changelog at the given time. -func (clv *Changelog[T]) Get(targetTime time.Time) T { - if len(clv.changes) == 0 { - return returnZero[T]() +// NewEntries initializes a new `Entries` structure using the provided `EntryFlagsMap`. +func NewEntries(m EntryFlagsMap) *Entries { + for _, v := range m { + if v.MaxEntries == 0 { + panic("EntryFlagsMap must have MaxEntries > 0") + } } - idx := clv.findIndex(targetTime) - if idx == 0 { - return returnZero[T]() + return &Entries{ + entryFlagsMap: m, + entries: []entry{}, } - - return clv.changes[idx-1].value } -// GetAll returns all the values of the changelog. -func (clv *Changelog[T]) GetAll() []T { - values := make([]T, 0, len(clv.changes)) - for _, change := range clv.changes { - values = append(values, change.value) +// Set adds or updates an entry in the changelog for the specified `MemberKind`. +// If the new entry has the same value as the latest one, only the timestamp is updated. +// If there are already the maximum number of entries for this kind, it reuses or replaces an existing entry. +// +// NOTE: Make sure to pass a value of the correct type for the specified `MemberKind`. +func (ce *Entries) Set(k MemberKind, value any, t time.Time) { + maxSize := int(ce.entryFlagsMap[k].MaxEntries) + if maxSize == 0 { + return } - return values -} -// Setters + indexes := make([]int, 0) + // collect indexes of entries equal to e + for idx, entry := range ce.entries { + if entry.k == k { + indexes = append(indexes, idx) + // last = false + } + } -// SetCurrent sets the latest value of the changelog. -func (clv *Changelog[T]) SetCurrent(value T) { - clv.setAt(value, time.Now()) -} + // if there are entries for e check if the last entry has the same value + if len(indexes) > 0 && isComparable(value) { + lastIdx := indexes[len(indexes)-1] + if ce.entries[lastIdx].value == value && t.After(ce.entries[lastIdx].t) { + // only update timestamp and return + ce.entries[lastIdx].t = t + return + } + } -// Set sets the value of the changelog at the given time. -func (clv *Changelog[T]) Set(value T, targetTime time.Time) { - clv.setAt(value, targetTime) -} + newEntry := entry{ + k: k, + t: t, + value: value, + } -// private + // + // if there is space, insert the new entry at the correct position + // -// setAt sets the value of the changelog at the given time. -func (clv *Changelog[T]) setAt(value T, targetTime time.Time) { - // If the timestamp is already set, update that value only. - _, ok := clv.timestamps[targetTime] - if ok { - index := clv.findIndex(targetTime) - 1 - if index < 0 { - logger.Debugw("changelog internal error: illegal index for existing timestamp") - } - if !clv.changes[index].timestamp.Equal(targetTime) { // sanity check only (time exists already) - logger.Debugw("changelog internal error: timestamp mismatch") + if len(indexes) < maxSize { + insertPos := ce.findInsertIdx(indexes, t) + if insertPos == len(ce.entries) { + ce.entries = append(ce.entries, newEntry) return } - if clv.changes[index].value != value { - logger.Debugw("changelog error: value mismatch for same timestamp") - } - clv.changes[index].value = value - return - } - entry := item[T]{ - timestamp: targetTime, - value: value, + ce.insertAt(insertPos, newEntry) + return } - idx := clv.findIndex(entry.timestamp) - // If the changelog has reached its maximum size and the new change would be inserted as the oldest, - // there is no need to add the new change. We can simply return without making any modifications. - if len(clv.changes) >= clv.maxSize && idx == 0 { - return + // + // as there is no space, replace a entry + // + + replaceIdx := indexes[len(indexes)-1] // index to replace + if t.After(ce.entries[replaceIdx].t) { + // reallocate values to the left + ce.shiftLeft(indexes) + } else { + // find the correct position to store the entry + replaceIdx = ce.findInsertIdx(indexes, t) - 1 + if replaceIdx == -1 { + replaceIdx = 0 + } } - // Insert the new entry in the changelog, keeping the list sorted by timestamp. - clv.changes = append(clv.changes, item[T]{}) - copy(clv.changes[idx+1:], clv.changes[idx:]) - clv.changes[idx] = entry - // Mark the timestamp as set. - clv.timestamps[targetTime] = struct{}{} - - clv.enforceSizeBoundary() + ce.entries[replaceIdx] = newEntry } -// findIndex returns the index of the first item in the changelog that is after the given time. -func (clv *Changelog[T]) findIndex(target time.Time) int { - left, right := 0, len(clv.changes) +// Get retrieves the value of the entry for the specified `MemberKind` at or before the given timestamp. +// If no matching entry is found, it returns the default value for the entry type. +func (ce *Entries) Get(k MemberKind, timestamp time.Time) any { + for i := len(ce.entries) - 1; i >= 0; i-- { + if ce.entries[i].k != k { + continue + } - for left < right { - middle := (left + right) / 2 - if clv.changes[middle].timestamp.After(target) { - right = middle - } else { - left = middle + 1 + if ce.entries[i].t.Before(timestamp) || ce.entries[i].t.Equal(timestamp) { + return ce.entries[i].value } } - return left + return ce.getZero(k) } -// enforceSizeBoundary ensures that the size of the inner array doesn't exceed the limit. -// It applies two methods to reduce the log size to the maximum allowed: -// 1. Unite duplicate values that are trailing one another, removing the oldest of the pair. -// 2. Remove the oldest logs as they are likely less important. - -func (clv *Changelog[T]) enforceSizeBoundary() { - if len(clv.changes) <= clv.maxSize { - // Nothing to do - return +// GetCurrent retrieves the most recent value for the specified `MemberKind`. +// If no entry is found, it returns the default value for the entry type. +func (ce *Entries) GetCurrent(k MemberKind) any { + for i := len(ce.entries) - 1; i >= 0; i-- { + if ce.entries[i].k == k { + return ce.entries[i].value + } } - boundaryDiff := len(clv.changes) - clv.maxSize - changed := false - - // Compact the slice in place - writeIdx := 0 - for readIdx := 0; readIdx < len(clv.changes); readIdx++ { - nextIdx := readIdx + 1 - if nextIdx < len(clv.changes) && - clv.changes[nextIdx].value == clv.changes[readIdx].value && - boundaryDiff > 0 { - // Remove the oldest (readIdx) from the duplicate pair - delete(clv.timestamps, clv.changes[readIdx].timestamp) - boundaryDiff-- - changed = true - continue - } + return ce.getZero(k) +} - // If elements have been removed or moved, update the map and the slice - if changed { - clv.changes[writeIdx] = clv.changes[readIdx] +// GetAll retrieves all values for the specified `MemberKind`, from the newest to the oldest. +func (ce *Entries) GetAll(k MemberKind) []any { + values := make([]any, 0) + for i := len(ce.entries) - 1; i >= 0; i-- { + if ce.entries[i].k == k { + values = append(values, ce.entries[i].value) } + } + + return values +} - writeIdx++ +// Count returns the number of entries recorded for the specified `MemberKind`. +func (ce *Entries) Count(k MemberKind) int { + count := 0 + for _, entry := range ce.entries { + if entry.k == k { + count++ + } } - if changed { - clear(clv.changes[writeIdx:]) - clv.changes = clv.changes[:writeIdx] + return count +} + +// findInsertIdx finds the correct index to insert a new entry based on its timestamp. +func (ce *Entries) findInsertIdx(indexes []int, t time.Time) int { + for i := len(indexes) - 1; i >= 0; i-- { + if ce.entries[indexes[i]].t.Before(t) { + return indexes[i] + 1 + } } - if len(clv.changes) <= clv.maxSize { - // Size is within limits after compaction - return + return len(indexes) +} + +// insertAt inserts a new entry at the specified index in the entries list. +func (ce *Entries) insertAt(idx int, newEntry entry) { + ce.entries = append(ce.entries[:idx], append([]entry{newEntry}, ce.entries[idx:]...)...) +} + +// shiftLeft shifts entries within the given indexes to the left, discarding the oldest entry. +func (ce *Entries) shiftLeft(indexes []int) { + for i := 0; i < len(indexes)-1; i++ { + ce.entries[indexes[i]] = ce.entries[indexes[i+1]] } +} - // As it still exceeds maxSize, remove the oldest entries in the remaining slice - boundaryDiff = len(clv.changes) - clv.maxSize - for i := 0; i < boundaryDiff; i++ { - delete(clv.timestamps, clv.changes[i].timestamp) +// getZero returns the default value for the specified `MemberKind` based on its type. +func (ce *Entries) getZero(k MemberKind) any { + t := ce.entryFlagsMap[k].Type + switch t { + case Byte: + return byte(0) + case Bool: + return false + case Int8: + return int8(0) + case Int16: + return int16(0) + case Int32: + return int32(0) + case Int64: + return int64(0) + case Int: + return int(0) + case Uint8: + return uint8(0) + case Uint16: + return uint16(0) + case Uint32: + return uint32(0) + case Uint64: + return uint64(0) + case Uint: + return uint(0) + case Float32: + return float32(0) + case Float64: + return float64(0) + case String: + return "" } - clear(clv.changes[:boundaryDiff]) - clv.changes = clv.changes[boundaryDiff:] + + // should never reach here if all cases are covered + return nil } -// returnZero returns the zero value of the type T. -func returnZero[T any]() T { - var zero T - return zero +// isComparable checks if a value can be compared. +func isComparable(value any) bool { + v := reflect.ValueOf(value) + return v.IsValid() && v.Type().Comparable() } diff --git a/pkg/changelog/changelog_benchmark_test.go b/pkg/changelog/changelog_benchmark_test.go index 2b349d9433e4..05f854cf331e 100644 --- a/pkg/changelog/changelog_benchmark_test.go +++ b/pkg/changelog/changelog_benchmark_test.go @@ -5,75 +5,6 @@ import ( "time" ) -func Benchmark_enforceSizeBoundary(b *testing.B) { - testCases := []struct { - name string - changelog Changelog[int] - }{ - { - name: "No change needed", - changelog: Changelog[int]{ - changes: []item[int]{ - {value: 1, timestamp: getTimeFromSec(1)}, - {value: 2, timestamp: getTimeFromSec(2)}, - }, - timestamps: map[time.Time]struct{}{ - getTimeFromSec(1): {}, - getTimeFromSec(2): {}, - }, - maxSize: 5, - }, - }, - { - name: "Trim excess with duplicates", - changelog: Changelog[int]{ - changes: []item[int]{ - {value: 1, timestamp: getTimeFromSec(1)}, - {value: 1, timestamp: getTimeFromSec(2)}, - {value: 2, timestamp: getTimeFromSec(3)}, - {value: 3, timestamp: getTimeFromSec(4)}, - {value: 3, timestamp: getTimeFromSec(5)}, - }, - timestamps: map[time.Time]struct{}{ - getTimeFromSec(1): {}, - getTimeFromSec(2): {}, - getTimeFromSec(3): {}, - getTimeFromSec(4): {}, - getTimeFromSec(5): {}, - }, - maxSize: 3, - }, - }, - { - name: "Remove oldest entries", - changelog: Changelog[int]{ - changes: []item[int]{ - {value: 1, timestamp: getTimeFromSec(1)}, - {value: 2, timestamp: getTimeFromSec(2)}, - {value: 3, timestamp: getTimeFromSec(3)}, - {value: 4, timestamp: getTimeFromSec(4)}, - }, - timestamps: map[time.Time]struct{}{ - getTimeFromSec(1): {}, - getTimeFromSec(2): {}, - getTimeFromSec(3): {}, - getTimeFromSec(4): {}, - }, - maxSize: 2, - }, - }, - } - - for _, tc := range testCases { - b.Run(tc.name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - clv := tc.changelog // Create a copy for each iteration - clv.enforceSizeBoundary() - } - }) - } -} - func Benchmark_Set(b *testing.B) { // Test cases where the Changelog needs to enforce the size boundary testCasesAllScenarios := []struct { @@ -142,13 +73,17 @@ func Benchmark_Set(b *testing.B) { }, } + entryFlagsMapAllScenarios := EntryFlagsMap{ + testInt0: {Type: Int, MaxEntries: 3}, + } + b.Run("All Scenarios", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - clv := NewChangelog[int](3) + clv := NewEntries(entryFlagsMapAllScenarios) b.StartTimer() for _, tc := range testCasesAllScenarios { - clv.Set(tc.value, tc.time) + clv.Set(testInt0, tc.value, tc.time) } } }) @@ -220,13 +155,17 @@ func Benchmark_Set(b *testing.B) { }, } + entryFlagsMapWithinLimit := EntryFlagsMap{ + testInt0: {Type: Int, MaxEntries: 15}, + } + b.Run("Within Limit", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - clv := NewChangelog[int](15) + clv := NewEntries(entryFlagsMapWithinLimit) b.StartTimer() for _, tc := range testCasesWithinLimit { - clv.Set(tc.value, tc.time) + clv.Set(testInt0, tc.value, tc.time) } } }) diff --git a/pkg/changelog/changelog_test.go b/pkg/changelog/changelog_test.go index f831aa7e5231..ea062298440d 100644 --- a/pkg/changelog/changelog_test.go +++ b/pkg/changelog/changelog_test.go @@ -1,350 +1,227 @@ package changelog import ( - "reflect" "testing" "time" "github.com/stretchr/testify/assert" ) -func TestChangelog(t *testing.T) { - t.Parallel() - - t.Run("GetCurrent on an empty changelog", func(t *testing.T) { - cl := NewChangelog[int](3) - - // Test GetCurrent on an empty changelog - assert.Zero(t, cl.GetCurrent()) - }) - - t.Run("Set and get", func(t *testing.T) { - cl := NewChangelog[int](3) - testVal := 42 - - cl.SetCurrent(testVal) - assert.Equal(t, testVal, cl.GetCurrent()) - }) - - t.Run("Set and get on set time", func(t *testing.T) { - cl := NewChangelog[int](3) - testVal1 := 42 - testVal2 := 76 - testVal3 := 76 - - // Test with 3 stages of the changelog to make sure the binary search works well for - // different lengths (both odd and even). - now := time.Now() - cl.Set(testVal1, now) - assert.Equal(t, testVal1, cl.Get(now)) - - cl.Set(testVal2, now.Add(time.Second)) - assert.Equal(t, testVal1, cl.Get(now)) - assert.Equal(t, testVal2, cl.Get(now.Add(time.Second))) - - cl.Set(testVal3, now.Add(2*time.Second)) - assert.Equal(t, testVal1, cl.Get(now)) - assert.Equal(t, testVal2, cl.Get(now.Add(time.Second))) - assert.Equal(t, testVal3, cl.Get(now.Add(2*time.Second))) - }) - - t.Run("Set twice on the same time", func(t *testing.T) { - cl := NewChangelog[int](3) - testVal := 42 - - now := time.Now() - cl.Set(testVal, now) - cl.Set(testVal, now) - assert.Equal(t, testVal, cl.Get(now)) - assert.Len(t, cl.GetAll(), 1) - assert.Equal(t, testVal, cl.Get(now)) - }) - - t.Run("Get on an empty changelog", func(t *testing.T) { - cl := NewChangelog[int](3) - - assert.Zero(t, cl.GetCurrent()) - }) - - t.Run("Test 1 second interval among changes", func(t *testing.T) { - cl := NewChangelog[int](3) - - cl.SetCurrent(1) - time.Sleep(2 * time.Second) - cl.SetCurrent(2) - time.Sleep(2 * time.Second) - cl.SetCurrent(3) - - now := time.Now() - - assert.Equal(t, 1, cl.Get(now.Add(-4*time.Second))) - assert.Equal(t, 2, cl.Get(now.Add(-2*time.Second))) - assert.Equal(t, 3, cl.Get(now)) - }) - - t.Run("Test 100 milliseconds interval among changes", func(t *testing.T) { - cl := NewChangelog[int](3) - - cl.SetCurrent(1) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(3) - - now := time.Now() - - assert.Equal(t, 1, cl.Get(now.Add(-200*time.Millisecond))) - assert.Equal(t, 2, cl.Get(now.Add(-100*time.Millisecond))) - assert.Equal(t, 3, cl.Get(now)) - }) - - t.Run("Test getting all values at once", func(t *testing.T) { - cl := NewChangelog[int](3) - - cl.SetCurrent(1) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(3) - - expected := []int{1, 2, 3} - assert.Equal(t, expected, cl.GetAll()) - }) - - t.Run("Pass max size wit repeated values", func(t *testing.T) { - cl := NewChangelog[int](3) - - cl.SetCurrent(1) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(3) - - now := time.Now() - assert.Equal(t, 1, cl.Get(now.Add(-300*time.Millisecond))) - assert.Equal(t, 1, cl.Get(now.Add(-200*time.Millisecond))) // oldest 2 is removed, so 1 is returned - assert.Equal(t, 2, cl.Get(now.Add(-100*time.Millisecond))) - assert.Equal(t, 3, cl.Get(now)) - assert.Len(t, cl.GetAll(), 3) - }) - - t.Run("Pass max size with unique values", func(t *testing.T) { - cl := NewChangelog[int](3) - - cl.SetCurrent(1) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(3) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(4) - - now := time.Now() - assert.Equal(t, 0, cl.Get(now.Add(-300*time.Millisecond))) - assert.Equal(t, 2, cl.Get(now.Add(-200*time.Millisecond))) - assert.Equal(t, 3, cl.Get(now.Add(-100*time.Millisecond))) - assert.Equal(t, 4, cl.Get(now.Add(time.Millisecond))) - assert.Len(t, cl.GetAll(), 3) - }) - - t.Run("Pass max size with new old value", func(t *testing.T) { - cl := NewChangelog[int](3) - - cl.SetCurrent(1) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(3) - - now := time.Now() - cl.Set(4, now.Add(-400*time.Millisecond)) - - // Make sure that the new value was not added - assert.Equal(t, 0, cl.Get(now.Add(-300*time.Millisecond))) - - // Sanity check - assert.Equal(t, 1, cl.Get(now.Add(-200*time.Millisecond))) - assert.Equal(t, 2, cl.Get(now.Add(-100*time.Millisecond))) - assert.Equal(t, 3, cl.Get(now)) - assert.Len(t, cl.GetAll(), 3) - }) - - t.Run("Zero sized changelog", func(t *testing.T) { - cl := NewChangelog[int](0) - - cl.SetCurrent(1) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(2) - time.Sleep(100 * time.Millisecond) - cl.SetCurrent(3) - - now := time.Now() - cl.Set(4, now.Add(-400*time.Millisecond)) - - // Make sure that the new value was not added - - // Sanity check - assert.Equal(t, 0, cl.Get(now.Add(-300*time.Millisecond))) - assert.Equal(t, 0, cl.Get(now.Add(-200*time.Millisecond))) - assert.Equal(t, 0, cl.Get(now.Add(-100*time.Millisecond))) - assert.Equal(t, 0, cl.Get(now)) - assert.Empty(t, cl.GetAll()) - }) - - t.Run("Test enforceSizeBoundary", func(t *testing.T) { - type TestCase struct { - name string - maxSize int - initialChanges []item[int] - expectedChanges []item[int] - expectedTimestamps map[time.Time]struct{} - } - - testCases := []TestCase{ - { - name: "No Action Required", - maxSize: 3, - initialChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 3}, - }, - expectedChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 3}, - }, - expectedTimestamps: map[time.Time]struct{}{ - getTimeFromSec(42): {}, - getTimeFromSec(43): {}, - getTimeFromSec(44): {}, - }, - }, - { - name: "Basic Removal of Oldest Entries", - maxSize: 3, - initialChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 3}, - {timestamp: getTimeFromSec(45), value: 4}, - }, - expectedChanges: []item[int]{ - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 3}, - {timestamp: getTimeFromSec(45), value: 4}, - }, - expectedTimestamps: map[time.Time]struct{}{ - getTimeFromSec(43): {}, - getTimeFromSec(44): {}, - getTimeFromSec(45): {}, - }, - }, - { - name: "Compacting Duplicate Values - Start", - maxSize: 3, - initialChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 1}, - {timestamp: getTimeFromSec(44), value: 2}, - {timestamp: getTimeFromSec(45), value: 3}, - }, - expectedChanges: []item[int]{ - {timestamp: getTimeFromSec(43), value: 1}, - {timestamp: getTimeFromSec(44), value: 2}, - {timestamp: getTimeFromSec(45), value: 3}, - }, - expectedTimestamps: map[time.Time]struct{}{ - getTimeFromSec(43): {}, - getTimeFromSec(44): {}, - getTimeFromSec(45): {}, - }, - }, - { - name: "Compacting Duplicate Values - Middle", - maxSize: 3, - initialChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 2}, - {timestamp: getTimeFromSec(45), value: 3}, - }, - expectedChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(44), value: 2}, - {timestamp: getTimeFromSec(45), value: 3}, - }, - expectedTimestamps: map[time.Time]struct{}{ - getTimeFromSec(42): {}, - getTimeFromSec(44): {}, - getTimeFromSec(45): {}, - }, - }, - { - name: "Compacting Duplicate Values - End", - maxSize: 3, - initialChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 3}, - {timestamp: getTimeFromSec(45), value: 3}, - }, - expectedChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(45), value: 3}, - }, - expectedTimestamps: map[time.Time]struct{}{ - getTimeFromSec(42): {}, - getTimeFromSec(43): {}, - getTimeFromSec(45): {}, - }, - }, - { - name: "Combination of Compaction and Removal of Oldest Entries", - maxSize: 3, - initialChanges: []item[int]{ - {timestamp: getTimeFromSec(42), value: 1}, - {timestamp: getTimeFromSec(43), value: 2}, - {timestamp: getTimeFromSec(44), value: 2}, - {timestamp: getTimeFromSec(45), value: 2}, - {timestamp: getTimeFromSec(46), value: 3}, - {timestamp: getTimeFromSec(47), value: 4}, - }, - expectedChanges: []item[int]{ - {timestamp: getTimeFromSec(45), value: 2}, - {timestamp: getTimeFromSec(46), value: 3}, - {timestamp: getTimeFromSec(47), value: 4}, - }, - expectedTimestamps: map[time.Time]struct{}{ - getTimeFromSec(45): {}, - getTimeFromSec(46): {}, - getTimeFromSec(47): {}, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cl := NewChangelog[int](tc.maxSize) - for _, change := range tc.initialChanges { - cl.Set(change.value, change.timestamp) - } - - cl.enforceSizeBoundary() - - eq := reflect.DeepEqual(cl.timestamps, tc.expectedTimestamps) - assert.True(t, eq) - - eq = reflect.DeepEqual(cl.changes, tc.expectedChanges) - assert.True(t, eq) - }) - } - }) -} +const ( + testInt0 MemberKind = iota + testInt1 + testInt2 + testString +) func getTimeFromSec(second int) time.Time { return time.Unix(int64(second), 0) } + +func TestChangelogEntries_MixedTypes(t *testing.T) { + flagsMap := EntryFlagsMap{ + testInt0: {Type: Int, MaxEntries: 2}, + testString: {Type: String, MaxEntries: 2}, + testInt1: {Type: Int, MaxEntries: 2}, + testInt2: {Type: Int, MaxEntries: 2}, + } + cl := NewEntries(flagsMap) + time0 := getTimeFromSec(0) + + // Set different value types + cl.Set(testInt0, 1234, time0) + cl.Set(testString, "process_name", time0) + cl.Set(testInt1, 5678, time0) + cl.Set(testInt2, 98765, time0) + + // Assert Get for each type + assert.Equal(t, 1234, cl.Get(testInt0, time0).(int), "Expected UID to be 1234") + assert.Equal(t, "process_name", cl.Get(testString, time0).(string), "Expected Name to be 'process_name'") + assert.Equal(t, 5678, cl.Get(testInt1, time0).(int), "Expected GID to be 5678") + assert.Equal(t, 98765, cl.Get(testInt2, time0).(int), "Expected NsPPid to be 98765") + + // Assert GetCurrent for each type + assert.Equal(t, 1234, cl.GetCurrent(testInt0).(int), "Expected current UID to be 1234") + assert.Equal(t, "process_name", cl.GetCurrent(testString).(string), "Expected current Name to be 'process_name'") + assert.Equal(t, 5678, cl.GetCurrent(testInt1).(int), "Expected current GID to be 5678") + assert.Equal(t, 98765, cl.GetCurrent(testInt2).(int), "Expected current NsPPid to be 98765") + + // Check the count of entries + assert.Equal(t, 1, cl.Count(testInt0), "Expected 1 entry for UID") + assert.Equal(t, 1, cl.Count(testString), "Expected 1 entry for Name") + assert.Equal(t, 1, cl.Count(testInt1), "Expected 1 entry for GID") + assert.Equal(t, 1, cl.Count(testInt2), "Expected 1 entry for NsPPid") +} + +func TestChangelogEntries_GetZeroValue(t *testing.T) { + flagsMap := EntryFlagsMap{ + testInt2: {Type: Int, MaxEntries: 1}, + testString: {Type: String, MaxEntries: 1}, + } + changelog := NewEntries(flagsMap) + time0 := getTimeFromSec(0) + + // Assert zero value before any set + assert.Equal(t, 0, changelog.Get(testInt2, time0), "Expected zero value for NsPPid") + assert.Equal(t, 0, changelog.GetCurrent(testInt2), "Expected zero value for current NsPPid") + + // Set and assert value + changelog.Set(testInt2, 3001, time0) + assert.Equal(t, 3001, changelog.Get(testInt2, time0), "Expected NsPPid to be 3001") + assert.Equal(t, 3001, changelog.GetCurrent(testInt2), "Expected current NsPPid to be 3001") + + // Check the count of entries + assert.Equal(t, 1, changelog.Count(testInt2), "Expected 1 entry") + assert.Equal(t, 0, changelog.Count(testString), "Expected 0 entries") +} + +func TestChangelogEntries_ShiftAndReplace(t *testing.T) { + flagsMap := EntryFlagsMap{ + testString: {Type: String, MaxEntries: 2}, + } + changelog := NewEntries(flagsMap) + + // Set entries and assert initial values + changelog.Set(testString, "initial", getTimeFromSec(0)) + changelog.Set(testString, "updated", getTimeFromSec(1)) + assert.Equal(t, "initial", changelog.Get(testString, getTimeFromSec(0)), "Expected first entry to be 'initial'") + assert.Equal(t, "updated", changelog.Get(testString, getTimeFromSec(1)), "Expected second entry to be 'updated'") + + // Test shifting and replacement + changelog.Set(testString, "final", getTimeFromSec(2)) + assert.Equal(t, "updated", changelog.Get(testString, getTimeFromSec(1)), "Expected oldest entry to be removed") + assert.Equal(t, "final", changelog.Get(testString, getTimeFromSec(2)), "Expected newest entry to be 'final'") + assert.Equal(t, "final", changelog.GetCurrent(testString), "Expected current entry to be 'final'") + + // Check the count of entries + assert.Equal(t, 2, changelog.Count(testString), "Expected 2 entries") +} + +func TestChangelogEntries_ReplaceMostRecentWithSameValue(t *testing.T) { + flagsMap := EntryFlagsMap{ + testString: {Type: String, MaxEntries: 2}, + } + changelog := NewEntries(flagsMap) + + // Set entries and assert initial value + changelog.Set(testString, "initial", getTimeFromSec(0)) + assert.Equal(t, "initial", changelog.Get(testString, getTimeFromSec(0)), "Expected first entry to be 'initial'") + changelog.Set(testString, "initial", getTimeFromSec(1)) + assert.Equal(t, "initial", changelog.Get(testString, getTimeFromSec(1)), "Expected first entry to have timestamp updated") + + // Test replacement of most recent entry with same value + changelog.Set(testString, "second", getTimeFromSec(2)) + assert.Equal(t, "initial", changelog.Get(testString, getTimeFromSec(1)), "Expected first entry to be 'initial'") + assert.Equal(t, "second", changelog.Get(testString, getTimeFromSec(2)), "Expected second entry to have timestamp updated") + + // Check the count of entries + assert.Equal(t, 2, changelog.Count(testString), "Expected 2 entries") +} + +func TestChangelogEntries_InsertWithOlderTimestamp(t *testing.T) { + flagsMap := EntryFlagsMap{ + testString: {Type: String, MaxEntries: 3}, + } + changelog := NewEntries(flagsMap) + now := getTimeFromSec(0) + + // Insert entries with increasing timestamps + changelog.Set(testString, "first", now) + changelog.Set(testString, "second", now.Add(1*time.Second)) + changelog.Set(testString, "third", now.Add(2*time.Second)) + + // Insert an entry with an older timestamp + changelog.Set(testString, "older", now.Add(1*time.Millisecond)) + + // Check the count of entries + assert.Equal(t, 3, changelog.Count(testString), "Expected 3 entries") + + // Verify the order of entries + assert.Equal(t, "older", changelog.Get(testString, now.Add(1*time.Millisecond)), "Expected 'older' to be the first entry") + assert.Equal(t, "second", changelog.Get(testString, now.Add(1*time.Second)), "Expected 'second' to be the second entry") + assert.Equal(t, "third", changelog.Get(testString, now.Add(2*time.Second)), "Expected 'third' to be the last entry") + + // Insert an entry with an intermediate timestamp + changelog.Set(testString, "second-third", now.Add(1*time.Second+1*time.Millisecond)) + + // Verify the order of entries + assert.Equal(t, "older", changelog.Get(testString, now.Add(1*time.Millisecond)), "Expected 'older' to be the first entry") + assert.Equal(t, "second-third", changelog.Get(testString, now.Add(1*time.Second+1*time.Millisecond)), "Expected 'second-third' to be the second entry") + assert.Equal(t, "third", changelog.Get(testString, now.Add(2*time.Second)), "Expected 'third' to be the last entry") + + // Check the count of entries + assert.Equal(t, 3, changelog.Count(testString), "Expected 3 entries") +} + +func TestChangelogEntries_InsertWithOlderTimestampAndMixedValues(t *testing.T) { + flagsMap := EntryFlagsMap{ + testString: {Type: String, MaxEntries: 3}, + testInt0: {Type: Int, MaxEntries: 3}, + } + changelog := NewEntries(flagsMap) + now := getTimeFromSec(0) + + // Insert entries with increasing timestamps + changelog.Set(testString, "first", now) + changelog.Set(testString, "second", now.Add(1*time.Second)) + changelog.Set(testString, "third", now.Add(2*time.Second)) + + changelog.Set(testInt0, 1001, now) + changelog.Set(testInt0, 1002, now.Add(1*time.Second)) + changelog.Set(testInt0, 1003, now.Add(2*time.Second)) + + // Check the count of entries + assert.Equal(t, 3, changelog.Count(testString), "Expected 3 entries") + assert.Equal(t, 3, changelog.Count(testInt0), "Expected 3 entries") + + // Insert an entry with an older timestamp + changelog.Set(testString, "older", now.Add(1*time.Millisecond)) + + // Verify the order of entries + assert.Equal(t, "older", changelog.Get(testString, now.Add(1*time.Millisecond)), "Expected 'older' to be the first entry") + assert.Equal(t, "second", changelog.Get(testString, now.Add(1*time.Second)), "Expected 'second' to be the second entry") + assert.Equal(t, "third", changelog.Get(testString, now.Add(2*time.Second)), "Expected 'third' to be the last entry") + + // Insert an entry with an intermediate timestamp + changelog.Set(testInt0, -1002, now.Add(1*time.Second+1*time.Millisecond)) + + // Verify the order of entries + assert.Equal(t, 1001, changelog.Get(testInt0, now), "Expected UID 1001 to be the first entry") + assert.Equal(t, -1002, changelog.Get(testInt0, now.Add(1*time.Second+1*time.Millisecond)), "Expected UID 1002 to be the second entry") + assert.Equal(t, 1003, changelog.Get(testInt0, now.Add(2*time.Second)), "Expected UID 1003 to be the last entry") + + // Check the count of entries + assert.Equal(t, 3, changelog.Count(testString), "Expected 3 entries") + assert.Equal(t, 3, changelog.Count(testInt0), "Expected 3 entries") +} + +func TestChangelogEntries_InsertSameValueWithNewTimestamp(t *testing.T) { + flagsMap := EntryFlagsMap{ + testString: {Type: String, MaxEntries: 3}, + } + changelog := NewEntries(flagsMap) + + // Insert entries with increasing timestamps + changelog.Set(testString, "same", getTimeFromSec(0)) + + // Replace the last entry with the same value but a new timestamp + changelog.Set(testString, "same", getTimeFromSec(1)) + + // Verify the order of entries + assert.Equal(t, "same", changelog.Get(testString, getTimeFromSec(1)), "Expected 'same' to be the second entry") + + // Insert entries with sequential timestamps + changelog.Set(testString, "new", getTimeFromSec(2)) + changelog.Set(testString, "other", getTimeFromSec(3)) + + // Replace the last entry with the same value but a new timestamp + changelog.Set(testString, "other", getTimeFromSec(4)) + + // Verify the order of entries + assert.Equal(t, "same", changelog.Get(testString, getTimeFromSec(1)), "Expected 'same' to be the first entry") + assert.Equal(t, "new", changelog.Get(testString, getTimeFromSec(2)), "Expected 'new' to be the second entry") + assert.Equal(t, "other", changelog.Get(testString, getTimeFromSec(4)), "Expected 'other' to be the last entry") + + // Check the count of entries + assert.Equal(t, 3, changelog.Count(testString), "Expected 3 entries") +} diff --git a/pkg/proctree/fileinfo.go b/pkg/proctree/fileinfo.go index 078b8a692711..311026a22939 100644 --- a/pkg/proctree/fileinfo.go +++ b/pkg/proctree/fileinfo.go @@ -4,7 +4,7 @@ import ( "sync" "time" - ch "github.com/aquasecurity/tracee/pkg/changelog" + "github.com/aquasecurity/tracee/pkg/changelog" ) // FileInfoFeed allows external packages to set/get multiple values of a task at once. @@ -21,32 +21,43 @@ type FileInfoFeed struct { // File Info // +const ( + fileInfoPath changelog.MemberKind = iota + fileInfoDev + fileInfoCtime + fileInfoInode + fileInfoInodeMode +) + +// fileInfoMutableMembersMap is a map with metadata about the mutable members of a FileInfo. +var fileInfoMutableMembersMap = changelog.EntryFlagsMap{ + fileInfoPath: {Type: changelog.String, MaxEntries: 3}, // file path + fileInfoDev: {Type: changelog.Int, MaxEntries: 3}, // device number of the file + fileInfoCtime: {Type: changelog.Int, MaxEntries: 3}, // creation time of the file + fileInfoInode: {Type: changelog.Int, MaxEntries: 3}, // inode number of the file + fileInfoInodeMode: {Type: changelog.Int, MaxEntries: 3}, // inode mode of the file +} + // FileInfo represents a file. type FileInfo struct { - path *ch.Changelog[string] // file path - dev *ch.Changelog[int] // device number of the file - ctime *ch.Changelog[int] // creation time of the file - inode *ch.Changelog[int] // inode number of the file - inodeMode *ch.Changelog[int] // inode mode of the file - mutex *sync.RWMutex + mutable *changelog.Entries + mutex *sync.RWMutex } // NewFileInfo creates a new file. -func NewFileInfo(maxLogSize int) *FileInfo { +func NewFileInfo() *FileInfo { return &FileInfo{ - path: ch.NewChangelog[string](maxLogSize), - dev: ch.NewChangelog[int](maxLogSize), - ctime: ch.NewChangelog[int](maxLogSize), - inode: ch.NewChangelog[int](maxLogSize), - inodeMode: ch.NewChangelog[int](maxLogSize), - mutex: &sync.RWMutex{}, + + mutable: changelog.NewEntries(fileInfoMutableMembersMap), + mutex: &sync.RWMutex{}, } } // NewFileInfoFeed creates a new file with values from the given feed. -func NewFileInfoFeed(maxLogSize int, feed FileInfoFeed) *FileInfo { - new := NewFileInfo(maxLogSize) +func NewFileInfoFeed(feed FileInfoFeed) *FileInfo { + new := NewFileInfo() new.SetFeed(feed) + return new } @@ -56,6 +67,7 @@ func NewFileInfoFeed(maxLogSize int, feed FileInfoFeed) *FileInfo { func (fi *FileInfo) SetFeed(feed FileInfoFeed) { fi.mutex.Lock() defer fi.mutex.Unlock() + fi.SetFeedAt(feed, time.Now()) } @@ -63,6 +75,7 @@ func (fi *FileInfo) SetFeed(feed FileInfoFeed) { func (fi *FileInfo) SetFeedAt(feed FileInfoFeed, targetTime time.Time) { fi.mutex.Lock() defer fi.mutex.Unlock() + fi.setFeedAt(feed, targetTime) } @@ -78,19 +91,19 @@ func (fi *FileInfo) setFeedAt(feed FileInfoFeed, targetTime time.Time) { // important parts. filePath = filePath[len(filePath)-MaxPathLen:] } - fi.path.Set(filePath, targetTime) + fi.mutable.Set(fileInfoPath, filePath, targetTime) } if feed.Dev >= 0 { - fi.dev.Set(feed.Dev, targetTime) + fi.mutable.Set(fileInfoDev, feed.Dev, targetTime) } if feed.Ctime >= 0 { - fi.ctime.Set(feed.Ctime, targetTime) + fi.mutable.Set(fileInfoCtime, feed.Ctime, targetTime) } if feed.Inode >= 0 { - fi.inode.Set(feed.Inode, targetTime) + fi.mutable.Set(fileInfoInode, feed.Inode, targetTime) } if feed.InodeMode >= 0 { - fi.inodeMode.Set(feed.InodeMode, targetTime) + fi.mutable.Set(fileInfoInodeMode, feed.InodeMode, targetTime) } } @@ -98,6 +111,7 @@ func (fi *FileInfo) setFeedAt(feed FileInfoFeed, targetTime time.Time) { func (fi *FileInfo) GetFeed() FileInfoFeed { fi.mutex.RLock() defer fi.mutex.RUnlock() + return fi.getFeedAt(time.Now()) } @@ -105,16 +119,17 @@ func (fi *FileInfo) GetFeed() FileInfoFeed { func (fi *FileInfo) GetFeedAt(targetTime time.Time) FileInfoFeed { fi.mutex.RLock() defer fi.mutex.RUnlock() + return fi.getFeedAt(targetTime) // return values at the given time } func (fi *FileInfo) getFeedAt(targetTime time.Time) FileInfoFeed { return FileInfoFeed{ - Path: fi.path.Get(targetTime), - Dev: fi.dev.Get(targetTime), - Ctime: fi.ctime.Get(targetTime), - Inode: fi.inode.Get(targetTime), - InodeMode: fi.inodeMode.Get(targetTime), + Path: fi.mutable.Get(fileInfoPath, targetTime).(string), + Dev: fi.mutable.Get(fileInfoDev, targetTime).(int), + Ctime: fi.mutable.Get(fileInfoCtime, targetTime).(int), + Inode: fi.mutable.Get(fileInfoInode, targetTime).(int), + InodeMode: fi.mutable.Get(fileInfoInodeMode, targetTime).(int), } } @@ -124,70 +139,80 @@ func (fi *FileInfo) getFeedAt(targetTime time.Time) FileInfoFeed { func (fi *FileInfo) SetPath(path string) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.path.Set(path, time.Now()) + + fi.mutable.Set(fileInfoPath, path, time.Now()) } // SetPathAt sets the path of the file at the given time. func (fi *FileInfo) SetPathAt(path string, targetTime time.Time) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.path.Set(path, targetTime) + + fi.mutable.Set(fileInfoPath, path, targetTime) } // SetDev sets the device number of the file. func (fi *FileInfo) SetDev(dev int) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.dev.Set(dev, time.Now()) + + fi.mutable.Set(fileInfoDev, dev, time.Now()) } // SetDevAt sets the device number of the file at the given time. func (fi *FileInfo) SetDevAt(dev int, targetTime time.Time) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.dev.Set(dev, targetTime) + + fi.mutable.Set(fileInfoDev, dev, targetTime) } // SetCtime sets the creation time of the file. func (fi *FileInfo) SetCtime(ctime int) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.ctime.Set(ctime, time.Now()) + + fi.mutable.Set(fileInfoCtime, ctime, time.Now()) } // SetCtimeAt sets the creation time of the file at the given time. func (fi *FileInfo) SetCtimeAt(ctime int, targetTime time.Time) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.ctime.Set(ctime, targetTime) + + fi.mutable.Set(fileInfoCtime, ctime, targetTime) } // SetInode sets the inode number of the file. func (fi *FileInfo) SetInode(inode int) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.inode.Set(inode, time.Now()) + + fi.mutable.Set(fileInfoInode, inode, time.Now()) } // SetInodeAt sets the inode number of the file at the given time. func (fi *FileInfo) SetInodeAt(inode int, targetTime time.Time) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.inode.Set(inode, targetTime) + + fi.mutable.Set(fileInfoInode, inode, targetTime) } // SetInodeMode sets the inode mode of the file. func (fi *FileInfo) SetInodeMode(inodeMode int) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.inodeMode.Set(inodeMode, time.Now()) + + fi.mutable.Set(fileInfoInodeMode, inodeMode, time.Now()) } // SetInodeModeAt sets the inode mode of the file at the given time. func (fi *FileInfo) SetInodeModeAt(inodeMode int, targetTime time.Time) { fi.mutex.Lock() defer fi.mutex.Unlock() - fi.inodeMode.Set(inodeMode, targetTime) + + fi.mutable.Set(fileInfoInodeMode, inodeMode, targetTime) } // Getters @@ -196,68 +221,78 @@ func (fi *FileInfo) SetInodeModeAt(inodeMode int, targetTime time.Time) { func (fi *FileInfo) GetPath() string { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.path.Get(time.Now()) + + return fi.mutable.Get(fileInfoPath, time.Now()).(string) } // GetPathAt returns the path of the file at the given time. func (fi *FileInfo) GetPathAt(targetTime time.Time) string { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.path.Get(targetTime) + + return fi.mutable.Get(fileInfoPath, targetTime).(string) } // GetDev returns the device number of the file. func (fi *FileInfo) GetDev() int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.dev.Get(time.Now()) + + return fi.mutable.Get(fileInfoDev, time.Now()).(int) } // GetDevAt returns the device number of the file at the given time. func (fi *FileInfo) GetDevAt(targetTime time.Time) int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.dev.Get(targetTime) + + return fi.mutable.Get(fileInfoDev, targetTime).(int) } // GetCtime returns the creation time of the file. func (fi *FileInfo) GetCtime() int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.ctime.Get(time.Now()) + + return fi.mutable.Get(fileInfoCtime, time.Now()).(int) } // GetCtimeAt returns the creation time of the file at the given time. func (fi *FileInfo) GetCtimeAt(targetTime time.Time) int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.ctime.Get(targetTime) + + return fi.mutable.Get(fileInfoCtime, targetTime).(int) } // GetInode returns the inode number of the file. func (fi *FileInfo) GetInode() int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.inode.Get(time.Now()) + + return fi.mutable.Get(fileInfoInode, time.Now()).(int) } // GetInodeAt returns the inode number of the file at the given time. func (fi *FileInfo) GetInodeAt(targetTime time.Time) int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.inode.Get(targetTime) + + return fi.mutable.Get(fileInfoInode, targetTime).(int) } // GetInodeMode returns the inode mode of the file. func (fi *FileInfo) GetInodeMode() int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.inodeMode.Get(time.Now()) + + return fi.mutable.Get(fileInfoInodeMode, time.Now()).(int) } // GetInodeModeAt returns the inode mode of the file at the given time. func (fi *FileInfo) GetInodeModeAt(targetTime time.Time) int { fi.mutex.RLock() defer fi.mutex.RUnlock() - return fi.inodeMode.Get(targetTime) + + return fi.mutable.Get(fileInfoInodeMode, targetTime).(int) } diff --git a/pkg/proctree/process.go b/pkg/proctree/process.go index 172176810090..7f6c54a24161 100644 --- a/pkg/proctree/process.go +++ b/pkg/proctree/process.go @@ -22,22 +22,15 @@ type Process struct { mutex *sync.RWMutex // mutex to protect the process } -const ( - executableChangelogSize = 5 // Binary's history is much more important to save - // TODO: Decide whether remove the interpreter and interp from the tree or add them back - interpreterChangelogSize = 0 - interpChangelogSize = 0 -) - // NewProcess creates a new process. func NewProcess(hash uint32) *Process { return &Process{ processHash: hash, parentHash: 0, info: NewTaskInfo(), - executable: NewFileInfo(executableChangelogSize), - interpreter: NewFileInfo(interpreterChangelogSize), - interp: NewFileInfo(interpChangelogSize), + executable: NewFileInfo(), + interpreter: NewFileInfo(), + interp: NewFileInfo(), children: make(map[uint32]struct{}), threads: make(map[uint32]struct{}), mutex: &sync.RWMutex{}, @@ -50,9 +43,9 @@ func NewProcessWithInfo(hash uint32, info *TaskInfo) *Process { processHash: hash, parentHash: 0, info: info, - executable: NewFileInfo(executableChangelogSize), - interpreter: NewFileInfo(interpreterChangelogSize), - interp: NewFileInfo(interpChangelogSize), + executable: NewFileInfo(), + interpreter: NewFileInfo(), + interp: NewFileInfo(), children: make(map[uint32]struct{}), threads: make(map[uint32]struct{}), mutex: &sync.RWMutex{}, diff --git a/pkg/proctree/taskinfo.go b/pkg/proctree/taskinfo.go index 15c5864238bf..612b753b5322 100644 --- a/pkg/proctree/taskinfo.go +++ b/pkg/proctree/taskinfo.go @@ -4,7 +4,7 @@ import ( "sync" "time" - ch "github.com/aquasecurity/tracee/pkg/changelog" + "github.com/aquasecurity/tracee/pkg/changelog" traceetime "github.com/aquasecurity/tracee/pkg/time" ) @@ -27,38 +27,45 @@ type TaskInfoFeed struct { // Task Info // +const ( + taskInfoName changelog.MemberKind = iota + taskInfoPPid + taskInfoNsPPid + taskInfoUid + taskInfoGid +) + +// taskInfoMutableMembersMap is a map with metadata about the mutable members of a TaskInfo. +var taskInfoMutableMembersMap = changelog.EntryFlagsMap{ + taskInfoName: {Type: changelog.String, MaxEntries: 3}, // process name can be changed + taskInfoPPid: {Type: changelog.Int, MaxEntries: 2}, // process can be reparented + taskInfoNsPPid: {Type: changelog.Int, MaxEntries: 2}, // process can be reparented + taskInfoUid: {Type: changelog.Int, MaxEntries: 2}, // process uid can be changed + taskInfoGid: {Type: changelog.Int, MaxEntries: 2}, // process gid can be changed +} + // TaskInfo represents a task. type TaskInfo struct { - name *ch.Changelog[string] // variable (process name can be changed) - tid int // immutable - pid int // immutable - pPid *ch.Changelog[int] // variable (process can be reparented) - nsTid int // immutable - nsPid int // immutable - nsPPid *ch.Changelog[int] // variable (process can be reparented) - uid *ch.Changelog[int] // variable (process uid can be changed) - gid *ch.Changelog[int] // variable (process gid can be changed) - startTimeNS uint64 // this is a duration, in ns, since boot (immutable) - exitTimeNS uint64 // this is a duration, in ns, since boot (immutable) + tid int // immutable + pid int // immutable + nsTid int // immutable + nsPid int // immutable + startTimeNS uint64 // this is a duration, in ns, since boot (immutable) + exitTimeNS uint64 // this is a duration, in ns, since boot (immutable) + mutable *changelog.Entries // variable fields mutex *sync.RWMutex } // NewTaskInfo creates a new task. func NewTaskInfo() *TaskInfo { return &TaskInfo{ - name: ch.NewChangelog[string](5), - // All the folloowing values changes are currently not monitored by the process tree. - // Hence, for now, they will only contain one value in the changelog - pPid: ch.NewChangelog[int](1), - nsPPid: ch.NewChangelog[int](1), - uid: ch.NewChangelog[int](1), - gid: ch.NewChangelog[int](1), - mutex: &sync.RWMutex{}, + mutable: changelog.NewEntries(taskInfoMutableMembersMap), + mutex: &sync.RWMutex{}, } } // NewTaskInfoFromFeed creates a new task with values from the given feed. -func NewTaskInfoFromFeed(feed TaskInfoFeed) *TaskInfo { +func NewTaskInfoNewFromFeed(feed TaskInfoFeed) *TaskInfo { new := NewTaskInfo() new.SetFeed(feed) return new @@ -70,6 +77,7 @@ func NewTaskInfoFromFeed(feed TaskInfoFeed) *TaskInfo { func (ti *TaskInfo) SetFeed(feed TaskInfoFeed) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.setFeedAt(feed, time.Now()) // set current values } @@ -77,12 +85,13 @@ func (ti *TaskInfo) SetFeed(feed TaskInfoFeed) { func (ti *TaskInfo) SetFeedAt(feed TaskInfoFeed, targetTime time.Time) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.setFeedAt(feed, targetTime) // set values at the given time } func (ti *TaskInfo) setFeedAt(feed TaskInfoFeed, targetTime time.Time) { if feed.Name != "" { - ti.name.Set(feed.Name, targetTime) + ti.mutable.Set(taskInfoName, feed.Name, targetTime) } if feed.Tid >= 0 { ti.tid = feed.Tid @@ -91,7 +100,7 @@ func (ti *TaskInfo) setFeedAt(feed TaskInfoFeed, targetTime time.Time) { ti.pid = feed.Pid } if feed.PPid >= 0 { - ti.pPid.Set(feed.PPid, targetTime) + ti.mutable.Set(taskInfoPPid, feed.PPid, targetTime) } if feed.NsTid >= 0 { ti.nsTid = feed.NsTid @@ -100,13 +109,13 @@ func (ti *TaskInfo) setFeedAt(feed TaskInfoFeed, targetTime time.Time) { ti.nsPid = feed.NsPid } if feed.NsPid >= 0 { - ti.nsPPid.Set(feed.NsPid, targetTime) + ti.mutable.Set(taskInfoNsPPid, feed.NsPPid, targetTime) } if feed.Uid >= 0 { - ti.uid.Set(feed.Uid, targetTime) + ti.mutable.Set(taskInfoUid, feed.Uid, targetTime) } if feed.Gid >= 0 { - ti.gid.Set(feed.Gid, targetTime) + ti.mutable.Set(taskInfoGid, feed.Gid, targetTime) } if feed.StartTimeNS != 0 { ti.startTimeNS = feed.StartTimeNS @@ -120,6 +129,7 @@ func (ti *TaskInfo) setFeedAt(feed TaskInfoFeed, targetTime time.Time) { func (ti *TaskInfo) GetFeed() TaskInfoFeed { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.getFeedAt(time.Now()) // return current values } @@ -127,22 +137,23 @@ func (ti *TaskInfo) GetFeed() TaskInfoFeed { func (ti *TaskInfo) GetFeedAt(targetTime time.Time) TaskInfoFeed { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.getFeedAt(targetTime) // return values at the given time } func (ti *TaskInfo) getFeedAt(targetTime time.Time) TaskInfoFeed { return TaskInfoFeed{ - Name: ti.name.Get(targetTime), Tid: ti.tid, Pid: ti.pid, - PPid: ti.pPid.Get(targetTime), NsTid: ti.nsTid, NsPid: ti.nsPid, - NsPPid: ti.nsPPid.Get(targetTime), - Uid: ti.uid.Get(targetTime), - Gid: ti.gid.Get(targetTime), StartTimeNS: ti.startTimeNS, ExitTimeNS: ti.exitTimeNS, + Name: ti.mutable.Get(taskInfoName, targetTime).(string), + PPid: ti.mutable.Get(taskInfoPPid, targetTime).(int), + NsPPid: ti.mutable.Get(taskInfoNsPPid, targetTime).(int), + Uid: ti.mutable.Get(taskInfoUid, targetTime).(int), + Gid: ti.mutable.Get(taskInfoGid, targetTime).(int), } } @@ -152,20 +163,23 @@ func (ti *TaskInfo) getFeedAt(targetTime time.Time) TaskInfoFeed { func (ti *TaskInfo) SetName(name string) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.name.Set(name, time.Now()) + + ti.mutable.Set(taskInfoName, name, time.Now()) } // SetNameAt sets the name of the task at the given time. func (ti *TaskInfo) SetNameAt(name string, targetTime time.Time) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.name.Set(name, targetTime) + + ti.mutable.Set(taskInfoName, name, targetTime) } // SetTid sets the tid of the task. func (ti *TaskInfo) SetTid(tid int) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.tid = tid } @@ -173,6 +187,7 @@ func (ti *TaskInfo) SetTid(tid int) { func (ti *TaskInfo) SetPid(pid int) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.pid = pid } @@ -180,6 +195,7 @@ func (ti *TaskInfo) SetPid(pid int) { func (ti *TaskInfo) SetNsTid(nsTid int) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.nsTid = nsTid } @@ -187,6 +203,7 @@ func (ti *TaskInfo) SetNsTid(nsTid int) { func (ti *TaskInfo) SetNsPid(nsPid int) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.nsPid = nsPid } @@ -194,6 +211,7 @@ func (ti *TaskInfo) SetNsPid(nsPid int) { func (ti *TaskInfo) SetStartTimeNS(startTimeNS uint64) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.startTimeNS = startTimeNS } @@ -201,6 +219,7 @@ func (ti *TaskInfo) SetStartTimeNS(startTimeNS uint64) { func (ti *TaskInfo) SetExitTime(exitTime uint64) { ti.mutex.Lock() defer ti.mutex.Unlock() + ti.exitTimeNS = exitTime } @@ -208,56 +227,64 @@ func (ti *TaskInfo) SetExitTime(exitTime uint64) { func (ti *TaskInfo) SetPPid(pPid int) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.pPid.Set(pPid, time.Now()) + + ti.mutable.Set(taskInfoPPid, pPid, time.Now()) } // SetPPidAt sets the ppid of the task at the given time. func (ti *TaskInfo) SetPPidAt(pPid int, targetTime time.Time) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.pPid.Set(pPid, targetTime) + + ti.mutable.Set(taskInfoPPid, pPid, targetTime) } // SetNsPPid sets the nsppid of the task. func (ti *TaskInfo) SetNsPPid(nsPPid int) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.nsPPid.Set(nsPPid, time.Now()) + + ti.mutable.Set(taskInfoNsPPid, nsPPid, time.Now()) } // SetNsPPidAt sets the nsppid of the task at the given time. func (ti *TaskInfo) SetNsPPidAt(nsPPid int, targetTime time.Time) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.nsPPid.Set(nsPPid, targetTime) + + ti.mutable.Set(taskInfoNsPPid, nsPPid, targetTime) } // SetUid sets the uid of the task. func (ti *TaskInfo) SetUid(uid int) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.uid.Set(uid, time.Now()) + + ti.mutable.Set(taskInfoUid, uid, time.Now()) } // SetUidAt sets the uid of the task at the given time. func (ti *TaskInfo) SetUidAt(uid int, targetTime time.Time) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.uid.Set(uid, targetTime) + + ti.mutable.Set(taskInfoUid, uid, targetTime) } // SetGid sets the gid of the task. func (ti *TaskInfo) SetGid(gid int) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.gid.Set(gid, time.Now()) + + ti.mutable.Set(taskInfoGid, gid, time.Now()) } // SetGidAt sets the gid of the task at the given time. func (ti *TaskInfo) SetGidAt(gid int, targetTime time.Time) { ti.mutex.Lock() defer ti.mutex.Unlock() - ti.gid.Set(gid, targetTime) + + ti.mutable.Set(taskInfoGid, gid, targetTime) } // Getters @@ -266,20 +293,23 @@ func (ti *TaskInfo) SetGidAt(gid int, targetTime time.Time) { func (ti *TaskInfo) GetName() string { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.name.GetCurrent() + + return ti.mutable.GetCurrent(taskInfoName).(string) } // GetNameAt returns the name of the task at the given time. func (ti *TaskInfo) GetNameAt(targetTime time.Time) string { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.name.Get(targetTime) + + return ti.mutable.Get(taskInfoName, targetTime).(string) } // GetTid returns the tid of the task. func (ti *TaskInfo) GetTid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.tid } @@ -287,6 +317,7 @@ func (ti *TaskInfo) GetTid() int { func (ti *TaskInfo) GetPid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.pid } @@ -294,6 +325,7 @@ func (ti *TaskInfo) GetPid() int { func (ti *TaskInfo) GetNsTid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.nsTid } @@ -301,6 +333,7 @@ func (ti *TaskInfo) GetNsTid() int { func (ti *TaskInfo) GetNsPid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.nsPid } @@ -308,62 +341,71 @@ func (ti *TaskInfo) GetNsPid() int { func (ti *TaskInfo) GetPPid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.pPid.GetCurrent() + + return ti.mutable.GetCurrent(taskInfoPPid).(int) } // GetPPidAt returns the ppid of the task at the given time. func (ti *TaskInfo) GetPPidAt(targetTime time.Time) int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.pPid.Get(targetTime) + + return ti.mutable.Get(taskInfoPPid, targetTime).(int) } // GetNsPPid returns the nsPPid of the task. func (ti *TaskInfo) GetNsPPid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.nsPPid.GetCurrent() + + return ti.mutable.GetCurrent(taskInfoNsPPid).(int) } // GetNsPPidAt returns the nsPPid of the task at the given time. func (ti *TaskInfo) GetNsPPidAt(targetTime time.Time) int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.nsPPid.Get(targetTime) + + return ti.mutable.Get(taskInfoNsPPid, targetTime).(int) } // GetUid returns the uid of the task. func (ti *TaskInfo) GetUid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.uid.GetCurrent() + + return ti.mutable.GetCurrent(taskInfoUid).(int) } // GetUidAt returns the uid of the task at the given time. func (ti *TaskInfo) GetUidAt(targetTime time.Time) int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.uid.Get(targetTime) + + return ti.mutable.Get(taskInfoUid, targetTime).(int) } // GetGid returns the gid of the task. func (ti *TaskInfo) GetGid() int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.gid.GetCurrent() + + return ti.mutable.GetCurrent(taskInfoGid).(int) } // GetGidAt returns the gid of the task at the given time. func (ti *TaskInfo) GetGidAt(targetTime time.Time) int { ti.mutex.RLock() defer ti.mutex.RUnlock() - return ti.gid.Get(targetTime) + + return ti.mutable.Get(taskInfoGid, targetTime).(int) } // GetStartTimeNS returns the start time of the task in nanoseconds since epoch func (ti *TaskInfo) GetStartTimeNS() uint64 { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.startTimeNS } @@ -379,6 +421,7 @@ func (ti *TaskInfo) GetStartTime() time.Time { func (ti *TaskInfo) GetExitTimeNS() uint64 { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.exitTimeNS } @@ -386,6 +429,7 @@ func (ti *TaskInfo) GetExitTimeNS() uint64 { func (ti *TaskInfo) GetExitTime() time.Time { ti.mutex.RLock() defer ti.mutex.RUnlock() + return traceetime.NsSinceEpochToTime(ti.exitTimeNS) } @@ -393,6 +437,7 @@ func (ti *TaskInfo) GetExitTime() time.Time { func (ti *TaskInfo) IsAlive() bool { ti.mutex.RLock() defer ti.mutex.RUnlock() + return ti.exitTimeNS == 0 } @@ -401,6 +446,7 @@ func (ti *TaskInfo) IsAlive() bool { func (ti *TaskInfo) IsAliveAt(targetTime time.Time) bool { ti.mutex.RLock() defer ti.mutex.RUnlock() + if ti.exitTimeNS != 0 { if targetTime.After(traceetime.NsSinceEpochToTime(ti.exitTimeNS)) { return false @@ -411,5 +457,6 @@ func (ti *TaskInfo) IsAliveAt(targetTime time.Time) bool { if targetTime.Before(traceetime.NsSinceEpochToTime(ti.startTimeNS)) { return false } + return true }