From dd9be2037fe67382d2bb9d3b541f2d3afaba30ff Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 2 Oct 2024 16:18:02 +0200 Subject: [PATCH 1/9] Default buffer type to 'heap' --- config/logstash.yml | 4 ++-- docs/static/config-details.asciidoc | 2 +- docs/static/settings-file.asciidoc | 4 ++-- logstash-core/lib/logstash/environment.rb | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index 62b5912c498..87151975beb 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -331,8 +331,8 @@ # pipeline.separate_logs: false # # Determine where to allocate memory buffers, for plugins that leverage them. -# Default to direct, optionally can be switched to heap to select Java heap space. -# pipeline.buffer.type: direct +# Default to heap, optionally can be switched to direct to select direct memory space. +# pipeline.buffer.type: heap # # ------------ X-Pack Settings (not applicable for OSS build)-------------- # diff --git a/docs/static/config-details.asciidoc b/docs/static/config-details.asciidoc index a68e89e2673..24e88857825 100644 --- a/docs/static/config-details.asciidoc +++ b/docs/static/config-details.asciidoc @@ -126,7 +126,7 @@ to provide better performance, especially when interacting with the network stac Under heavy load, namely large number of connections and large messages, the direct memory space can be exhausted and lead to Out of Memory (OOM) errors in off-heap space. An off-heap OOM is difficult to debug, so {ls} provides a `pipeline.buffer.type` setting in <> that lets you control where to allocate memory buffers for plugins that use them. -Currently it is set to `direct` by default, but you can change it to `heap` to use Java heap space instead, which will be become the default in the future. +Currently it is set to `heap` by default, but you can change it to `direct` to use direct memory space instead. When set to `heap`, buffer allocations used by plugins are configured to **prefer** the Java Heap instead of direct memory, as direct memory allocations may still be necessary depending on the plugin. diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index d06327cce3d..942af1d40de 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -369,6 +369,6 @@ Setting this flag to `warn` is deprecated and will be removed in a future releas | `pipeline.buffer.type` | Determine where to allocate memory buffers, for plugins that leverage them. -Currently defaults to `direct` but can be switched to `heap` to select Java heap space, which will become the default in the future. -| `direct` Check out <> for more info. +Currently defaults to `heap` but can be switched to `direct` to select direct memory space. +| `heap` Check out <> for more info. |======================================================================= diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 164a190eb69..8d796950065 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -111,7 +111,7 @@ module Environment Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"), Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on Setting::NullableString.new("monitoring.cluster_uuid"), - Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]) + Setting::String.new("pipeline.buffer.type", "heap", true, ["direct", "heap"]) # post_process ].each {|setting| SETTINGS.register(setting) } From a9aa357ddfb91945646598bc03ea85a9e6c07b41 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 9 Oct 2024 17:17:55 +0200 Subject: [PATCH 2/9] [Test] Fixed x-pack tests of StatsEventFactory to verify the web server binding iff the webserver setting is enabled --- .../monitoring/inputs/metrics/stats_event_factory_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb b/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb index b37b13a26af..011d74b6e92 100644 --- a/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb +++ b/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb @@ -95,7 +95,11 @@ context "new model" do context "with webserver disabled" do it_behaves_like("new model monitoring event with webserver setting") do +<<<<<<< HEAD let(:webserver_enabled) {false} +======= + let(:webserver_enabled) {false} +>>>>>>> df87295f0 ([Test] Fixed x-pack tests of StatsEventFactory to verify the web server binding iff the webserver setting is enabled) end end From 19d8586c3f6439b3b3df3635497274bbe8d9c6da Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 10 Oct 2024 19:07:59 +0200 Subject: [PATCH 3/9] Reverted changes covered by #16525 --- .../monitoring/inputs/metrics/stats_event_factory_spec.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb b/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb index 011d74b6e92..cbe5309445a 100644 --- a/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb +++ b/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb @@ -50,7 +50,6 @@ let(:pipeline_settings) { LogStash::Runner::SYSTEM_SETTINGS.clone.merge({ "pipeline.id" => "main", "config.string" => config, - "api.enabled" => webserver_enabled, }) } let(:agent) { LogStash::Agent.new(pipeline_settings) } @@ -95,11 +94,7 @@ context "new model" do context "with webserver disabled" do it_behaves_like("new model monitoring event with webserver setting") do -<<<<<<< HEAD let(:webserver_enabled) {false} -======= - let(:webserver_enabled) {false} ->>>>>>> df87295f0 ([Test] Fixed x-pack tests of StatsEventFactory to verify the web server binding iff the webserver setting is enabled) end end From 41376d287b121bf1801b8173b1b47bab46dbf706 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 10 Oct 2024 19:14:02 +0200 Subject: [PATCH 4/9] Re-add missed during rebase --- .../spec/monitoring/inputs/metrics/stats_event_factory_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb b/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb index cbe5309445a..b37b13a26af 100644 --- a/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb +++ b/x-pack/spec/monitoring/inputs/metrics/stats_event_factory_spec.rb @@ -50,6 +50,7 @@ let(:pipeline_settings) { LogStash::Runner::SYSTEM_SETTINGS.clone.merge({ "pipeline.id" => "main", "config.string" => config, + "api.enabled" => webserver_enabled, }) } let(:agent) { LogStash::Agent.new(pipeline_settings) } From 44fdfaf82383e47058cd30d9caeebc34262ac6b4 Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 11 Oct 2024 15:20:17 +0200 Subject: [PATCH 5/9] Reworked the section that describe the off-heap allocation, to just describe the possibility to switch back to direct allocation in case the user want the previous behaviour --- docs/static/config-details.asciidoc | 34 ++++++++++------------------- docs/static/settings-file.asciidoc | 2 +- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/docs/static/config-details.asciidoc b/docs/static/config-details.asciidoc index 24e88857825..da97a873ee2 100644 --- a/docs/static/config-details.asciidoc +++ b/docs/static/config-details.asciidoc @@ -118,30 +118,20 @@ To summarize, we have 3 categories of memory usage, where 2 can be limited by th Keep these memory requirements in mind as you calculate your ideal memory allocation. -[[reducing-off-heap-usage]] -===== Upcoming changes to Buffer Allocation and Troubleshooting Out of Memory errors - -Plugins such as {agent}, {beats}, TCP, and HTTP inputs, currently default to using direct memory as it tends -to provide better performance, especially when interacting with the network stack. -Under heavy load, namely large number of connections and large messages, the direct memory space can be exhausted and lead to Out of Memory (OOM) errors in off-heap space. - -An off-heap OOM is difficult to debug, so {ls} provides a `pipeline.buffer.type` setting in <> that lets you control where to allocate memory buffers for plugins that use them. +[[off-heap-buffers-allocation]] +===== Buffer Allocation types +Plugins such as {agent}, {beats}, TCP, and HTTP inputs, use by default Java heap memory to allocate the buffers +used to read events from the network. This is the preferred allocation type because permit to use better tools +to investigate potential issues with leaking of memory that would result in Out of Memory (OOM) errors. + +However, in the past the default allocation was in direct memory, to re-enable the old behaviour {ls} provides +a `pipeline.buffer.type` setting in <> that lets you control where to allocate +memory buffers for plugins that use them. Currently it is set to `heap` by default, but you can change it to `direct` to use direct memory space instead. -When set to `heap`, buffer allocations used by plugins are configured to **prefer** the -Java Heap instead of direct memory, as direct memory allocations may still be necessary depending on the plugin. - -When set to "heap", in the event of an out-of-memory, Logstash will produce a heap dump to facilitate debugging. - -It is important to note that the Java heap sizing requirements will be impacted by this change since -allocations that previously resided on the direct memory will use heap instead. - -Performance-wise there shouldn't be a noticeable impact, since while direct memory IO is faster, Logstash Event objects produced by these plugins end up being allocated on the Java Heap, incurring the cost of copying from direct memory to heap memory regardless of the setting. -[NOTE] --- -* When you set `pipeline.buffer.type` to `heap`, consider incrementing the Java heap by the -amount of memory that had been reserved for direct space. --- +Performance-wise there shouldn't be a noticeable impact in switching to `direct`, since while direct memory IO is faster, +Logstash Event objects produced by these plugins end up being allocated on the Java Heap, +incurring the cost of copying from direct memory to heap memory regardless of the setting. [[memory-size-calculation]] ===== Memory sizing diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index 942af1d40de..8a75a759684 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -370,5 +370,5 @@ Setting this flag to `warn` is deprecated and will be removed in a future releas | `pipeline.buffer.type` | Determine where to allocate memory buffers, for plugins that leverage them. Currently defaults to `heap` but can be switched to `direct` to select direct memory space. -| `heap` Check out <> for more info. +| `heap` Check out <> for more info. |======================================================================= From 9bea3e5147af8be43718382e093419c8f96d53c9 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Thu, 17 Oct 2024 09:12:29 +0200 Subject: [PATCH 6/9] Better wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: João Duarte --- config/logstash.yml | 2 +- docs/static/config-details.asciidoc | 5 ++--- docs/static/settings-file.asciidoc | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index 87151975beb..6315577c622 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -331,7 +331,7 @@ # pipeline.separate_logs: false # # Determine where to allocate memory buffers, for plugins that leverage them. -# Default to heap, optionally can be switched to direct to select direct memory space. +# Defaults to heap, optionally can be switched to direct to select direct memory space. # pipeline.buffer.type: heap # # ------------ X-Pack Settings (not applicable for OSS build)-------------- diff --git a/docs/static/config-details.asciidoc b/docs/static/config-details.asciidoc index da97a873ee2..0897b153fe5 100644 --- a/docs/static/config-details.asciidoc +++ b/docs/static/config-details.asciidoc @@ -120,9 +120,8 @@ Keep these memory requirements in mind as you calculate your ideal memory alloca [[off-heap-buffers-allocation]] ===== Buffer Allocation types -Plugins such as {agent}, {beats}, TCP, and HTTP inputs, use by default Java heap memory to allocate the buffers -used to read events from the network. This is the preferred allocation type because permit to use better tools -to investigate potential issues with leaking of memory that would result in Out of Memory (OOM) errors. +Input plugins such as {agent}, {beats}, TCP, and HTTP will allocate buffers in Java heap memory to read events from the network. +This is the preferred allocation method as it facilitates debugging memory usage problems (such as leaks and Out of Memory errors) through the analysis of heap dumps. However, in the past the default allocation was in direct memory, to re-enable the old behaviour {ls} provides a `pipeline.buffer.type` setting in <> that lets you control where to allocate diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index 8a75a759684..2293e96f6fa 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -369,6 +369,6 @@ Setting this flag to `warn` is deprecated and will be removed in a future releas | `pipeline.buffer.type` | Determine where to allocate memory buffers, for plugins that leverage them. -Currently defaults to `heap` but can be switched to `direct` to select direct memory space. +Defaults to `heap` but can be switched to `direct` to instruct Logstash to prefer allocation of buffers in direct memory. | `heap` Check out <> for more info. |======================================================================= From a9098a42ca5e7db3d914f1be48ff3f9d697a3166 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 17 Oct 2024 09:19:50 +0200 Subject: [PATCH 7/9] Merged multiple suggestion on a phrase to shape it better --- docs/static/config-details.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/static/config-details.asciidoc b/docs/static/config-details.asciidoc index 0897b153fe5..cec39d5cd13 100644 --- a/docs/static/config-details.asciidoc +++ b/docs/static/config-details.asciidoc @@ -123,10 +123,9 @@ Keep these memory requirements in mind as you calculate your ideal memory alloca Input plugins such as {agent}, {beats}, TCP, and HTTP will allocate buffers in Java heap memory to read events from the network. This is the preferred allocation method as it facilitates debugging memory usage problems (such as leaks and Out of Memory errors) through the analysis of heap dumps. -However, in the past the default allocation was in direct memory, to re-enable the old behaviour {ls} provides +Before version 9.0.0, Logstash defaulted to allocate direct memory for this purpose, instead of heap. To re-enable the previous behaviour {ls} provides a `pipeline.buffer.type` setting in <> that lets you control where to allocate memory buffers for plugins that use them. -Currently it is set to `heap` by default, but you can change it to `direct` to use direct memory space instead. Performance-wise there shouldn't be a noticeable impact in switching to `direct`, since while direct memory IO is faster, Logstash Event objects produced by these plugins end up being allocated on the Java Heap, From 31788c3e7cae59037371df71efd255ae883a4ee6 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Fri, 18 Oct 2024 08:52:24 +0200 Subject: [PATCH 8/9] Update docs/static/config-details.asciidoc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: João Duarte --- docs/static/config-details.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/static/config-details.asciidoc b/docs/static/config-details.asciidoc index cec39d5cd13..d15802ca979 100644 --- a/docs/static/config-details.asciidoc +++ b/docs/static/config-details.asciidoc @@ -123,7 +123,7 @@ Keep these memory requirements in mind as you calculate your ideal memory alloca Input plugins such as {agent}, {beats}, TCP, and HTTP will allocate buffers in Java heap memory to read events from the network. This is the preferred allocation method as it facilitates debugging memory usage problems (such as leaks and Out of Memory errors) through the analysis of heap dumps. -Before version 9.0.0, Logstash defaulted to allocate direct memory for this purpose, instead of heap. To re-enable the previous behaviour {ls} provides +Before version 9.0.0, Logstash defaulted to allocate direct memory for this purpose instead of heap. To re-enable the previous behaviour {ls} provides a `pipeline.buffer.type` setting in <> that lets you control where to allocate memory buffers for plugins that use them. From d64d8349e6636f4508f3e798ddb6ed702294c057 Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 18 Oct 2024 08:56:09 +0200 Subject: [PATCH 9/9] Applied Joao rephrasing suggestion --- docs/static/config-details.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/static/config-details.asciidoc b/docs/static/config-details.asciidoc index d15802ca979..a76e9ed46fb 100644 --- a/docs/static/config-details.asciidoc +++ b/docs/static/config-details.asciidoc @@ -127,9 +127,9 @@ Before version 9.0.0, Logstash defaulted to allocate direct memory for this purp a `pipeline.buffer.type` setting in <> that lets you control where to allocate memory buffers for plugins that use them. -Performance-wise there shouldn't be a noticeable impact in switching to `direct`, since while direct memory IO is faster, -Logstash Event objects produced by these plugins end up being allocated on the Java Heap, -incurring the cost of copying from direct memory to heap memory regardless of the setting. +There shouldn't be a noticeable performance impact in switching between `direct` and `heap`. +While copying bytes from OS buffers to direct memory buffers is faster, Logstash Event objects produced by these plugins end up +being allocated on the Java Heap incurring the cost of copying from direct memory to heap memory, regardless of the setting. [[memory-size-calculation]] ===== Memory sizing