From f1439bc42e8a9498b169c14eb030d8d6f2530ac8 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 11 Apr 2024 08:57:13 -0500 Subject: [PATCH] enhancement(source metrics): Adding a histogram for event byte size (#19686) * Adding a histogram for event byte size. Useful when debugging low-level sources like UDP, TCP sockets * Adding documentation and fixing CI * fixing things up * Making sure only low-level sources have event size histograms (especially those based on UDP). * Add docs fixes * cargo fmt * fixup --- changelog.d/19686_element_size_histogram.enhancement.md | 7 +++++++ src/internal_events/socket.rs | 4 +++- .../cue/reference/components/sources/internal_metrics.cue | 6 ++++++ website/cue/reference/components/sources/socket.cue | 1 + website/cue/reference/components/sources/statsd.cue | 3 +++ website/cue/reference/components/sources/syslog.cue | 1 + 6 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19686_element_size_histogram.enhancement.md diff --git a/changelog.d/19686_element_size_histogram.enhancement.md b/changelog.d/19686_element_size_histogram.enhancement.md new file mode 100644 index 0000000000000..fa1ed69ffb90a --- /dev/null +++ b/changelog.d/19686_element_size_histogram.enhancement.md @@ -0,0 +1,7 @@ +Added a new histogram metric, `component_received_bytes`, that measures the byte-size of individual events received by the following sources: + +- `socket` +- `statsd` +- `syslog` + +authors: pabloem diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index 157084d52f340..10a144c8eed22 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,4 +1,4 @@ -use metrics::counter; +use metrics::{counter, histogram}; use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; use vector_lib::{ internal_event::{error_stage, error_type}, @@ -40,6 +40,7 @@ impl InternalEvent for SocketBytesReceived { "component_received_bytes_total", self.byte_size as u64, "protocol" => protocol, ); + histogram!("component_received_bytes", self.byte_size as f64); } } @@ -61,6 +62,7 @@ impl InternalEvent for SocketEventsReceived { ); counter!("component_received_events_total", self.count as u64, "mode" => mode); counter!("component_received_event_bytes_total", self.byte_size.get() as u64, "mode" => mode); + histogram!("component_received_bytes", self.byte_size.get() as f64, "mode" => mode); } } diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 2e31b74d97628..485ef60903d9e 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -370,6 +370,12 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: component_received_events_total.tags } + component_received_bytes: { + description: string | *"The size in bytes of each event received by the source." + type: "histogram" + default_namespace: "vector" + tags: component_received_events_total.tags + } component_received_events_total: { description: """ The number of events accepted by this component either from tagged diff --git a/website/cue/reference/components/sources/socket.cue b/website/cue/reference/components/sources/socket.cue index 808b94834ff9c..a417ba8c065cb 100644 --- a/website/cue/reference/components/sources/socket.cue +++ b/website/cue/reference/components/sources/socket.cue @@ -114,5 +114,6 @@ components: sources: socket: { connection_established_total: components.sources.internal_metrics.output.metrics.connection_established_total connection_send_errors_total: components.sources.internal_metrics.output.metrics.connection_send_errors_total connection_shutdown_total: components.sources.internal_metrics.output.metrics.connection_shutdown_total + component_received_bytes: components.sources.internal_metrics.output.metrics.component_received_bytes } } diff --git a/website/cue/reference/components/sources/statsd.cue b/website/cue/reference/components/sources/statsd.cue index 4f378e889d97c..f072190372d99 100644 --- a/website/cue/reference/components/sources/statsd.cue +++ b/website/cue/reference/components/sources/statsd.cue @@ -84,4 +84,7 @@ components: sources: statsd: { """ } } + telemetry: metrics: { + component_received_bytes: components.sources.internal_metrics.output.metrics.component_received_bytes + } } diff --git a/website/cue/reference/components/sources/syslog.cue b/website/cue/reference/components/sources/syslog.cue index 9efb9c09f0902..51764945f81a8 100644 --- a/website/cue/reference/components/sources/syslog.cue +++ b/website/cue/reference/components/sources/syslog.cue @@ -213,5 +213,6 @@ components: sources: syslog: { telemetry: metrics: { connection_read_errors_total: components.sources.internal_metrics.output.metrics.connection_read_errors_total utf8_convert_errors_total: components.sources.internal_metrics.output.metrics.utf8_convert_errors_total + component_received_bytes: components.sources.internal_metrics.output.metrics.component_received_bytes } }