From 4fddbab0aae14198db05e476636829ba8167aedc Mon Sep 17 00:00:00 2001 From: Vadim Alekseev Date: Mon, 24 Feb 2025 12:40:05 +0300 Subject: [PATCH] Use "low_memory' pool as default event pool (#787) Use "low_memory' pool as default event pool --- pipeline/README.idoc.md | 17 ++++++++++++++++- pipeline/README.md | 17 ++++++++++++++++- pipeline/pipeline.go | 7 +++---- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/pipeline/README.idoc.md b/pipeline/README.idoc.md index 2390cc0e6..9b2652fea 100644 --- a/pipeline/README.idoc.md +++ b/pipeline/README.idoc.md @@ -112,7 +112,22 @@ The amount of time the metric can be idle until it is deleted. Used for deleting **`pool`** *`string`* *`options=std|low_memory`* -Type of `EventPool`. `std` pool is an original pool with the slice of `Event` pointers and slices of free events indicators. `low_memory` pool is a leveled pool based on multiple `sync.Pool` for the events of different size. The latter one is experimental. +Type of `EventPool` that file.d uses to reuse memory. +`std` pool is an original event pool with pre-allocated events at the start of the application. +This pool only frees up memory if event exceeds `avg_log_size`. + +`low_memory` event pool based on Go's [sync.Pool](https://pkg.go.dev/sync#Pool) with lazy memory allocation. +It frees up memory depending on the application load - if file.d processes a lot of events, then a lot of memory will be allocated. +If the application load decreases, then the extra events will be freed up in background. + +Note that `low_memory` pool increases the load on the garbage collector. +If you are confident in what you are doing, you can change the [GOGC](https://tip.golang.org/doc/gc-guide) (file.d uses GOGC=30 as default value) +environment variable to adjust the frequency of garbage collection – this can reduce the load on the CPU. + +Both pools support the `capacity` setting, which both pools use to ensure that they do not exceed the number of allocated events. +This parameter is useful for avoiding OOM. + +Default pool is `low_memory`.
diff --git a/pipeline/README.md b/pipeline/README.md index 7c7aea0ac..dc8b52716 100755 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -112,7 +112,22 @@ The amount of time the metric can be idle until it is deleted. Used for deleting **`pool`** *`string`* *`options=std|low_memory`* -Type of `EventPool`. `std` pool is an original pool with the slice of `Event` pointers and slices of free events indicators. `low_memory` pool is a leveled pool based on multiple `sync.Pool` for the events of different size. The latter one is experimental. +Type of `EventPool` that file.d uses to reuse memory. +`std` pool is an original event pool with pre-allocated events at the start of the application. +This pool only frees up memory if event exceeds `avg_log_size`. + +`low_memory` event pool based on Go's [sync.Pool](https://pkg.go.dev/sync#Pool) with lazy memory allocation. +It frees up memory depending on the application load - if file.d processes a lot of events, then a lot of memory will be allocated. +If the application load decreases, then the extra events will be freed up in background. + +Note that `low_memory` pool increases the load on the garbage collector. +If you are confident in what you are doing, you can change the [GOGC](https://tip.golang.org/doc/gc-guide) (file.d uses GOGC=30 as default value) +environment variable to adjust the frequency of garbage collection – this can reduce the load on the CPU. + +Both pools support the `capacity` setting, which both pools use to ensure that they do not exceed the number of allocated events. +This parameter is useful for avoiding OOM. + +Default pool is `low_memory`.
diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 36d58ae79..a1eacfcc8 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -34,7 +34,7 @@ const ( DefaultMaxInputEventSize = 0 DefaultCutOffEventByLimit = false DefaultCutOffEventByLimitField = "" - DefaultJSONNodePoolSize = 1024 + DefaultJSONNodePoolSize = 16 DefaultMaintenanceInterval = time.Second * 5 DefaultEventTimeout = time.Second * 30 DefaultFieldValue = "not_set" @@ -181,10 +181,9 @@ func New(name string, settings *Settings, registry *prometheus.Registry) *Pipeli var eventPool pool switch settings.Pool { - case PoolTypeStd, "": + case PoolTypeStd: eventPool = newEventPool(settings.Capacity, settings.AvgEventSize) - case PoolTypeLowMem: - insaneJSON.StartNodePoolSize = 16 + case PoolTypeLowMem, "": eventPool = newLowMemoryEventPool(settings.Capacity) default: logger.Fatal("unknown pool type", zap.String("pool", string(settings.Pool)))