[Amazon Security Lake] - OCSF v1.1 update with major refactor & adding support for dynamic template and mappings & system tests

packages/amazon_security_lake/data_stream/application_activity/fields/actor-fields.yml

@@ -196,6 +196,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The username. For example, janedoe1.
@@ -285,6 +288,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The name of the city.
@@ -383,6 +389,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The username. For example, janedoe1.
@@ -457,6 +466,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The username. For example, janedoe1.
@@ -782,6 +794,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The username. For example, janedoe1.
@@ -870,6 +885,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The name of the city.
@@ -967,6 +985,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The username. For example, janedoe1.
@@ -1040,6 +1061,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The username. For example, janedoe1.
@@ -1270,6 +1294,9 @@
                     - name: uuid
                       type: keyword
                       description: The universally unique identifier of the session.
+                    - name: terminal
+                      type: keyword
+                      description: The Pseudo Terminal associated with the session. Ex, the tty or pts value.
                 - name: terminated_time
                   type: date
                   description: The time when the process was terminated.
@@ -1397,6 +1424,9 @@
                 - name: uuid
                   type: keyword
                   description: The universally unique identifier of the session.
+                - name: terminal
+                  type: keyword
+                  description: The Pseudo Terminal associated with the session. Ex, the tty or pts value.
             - name: terminated_time
               type: date
               description: The time when the process was terminated.

packages/amazon_security_lake/data_stream/application_activity/fields/device-fields.yml

@@ -34,6 +34,9 @@
             - name: desc
               type: keyword
               description: The group description.
+            - name: domain
+              type: keyword
+              description: The domain where the group is defined. For example, the LDAP or Active Directory domain.
             - name: name
               type: keyword
               description: The group name.

packages/amazon_security_lake/data_stream/application_activity/fields/fields.yml

@@ -257,6 +257,7 @@
       fields:
         - name: data
           type: flattened
+          ignore_malformed: true
           description: The enrichment data associated with the attribute and value. The meaning of this data depends on the type the enrichment record.
         - name: name
           type: keyword

packages/amazon_security_lake/data_stream/application_activity/manifest.yml

@@ -1,3 +1,9 @@
 title: Amazon Security Lake Application Activity Events
 dataset: amazon_security_lake.application_activity
 type: logs
+elasticsearch:
+  dynamic_dataset: true
+  dynamic_namespace: true
+  index_template:
+    mappings:
+      dynamic: true

packages/amazon_security_lake/data_stream/discovery/fields/actor-fields.yml

@@ -196,6 +196,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The username. For example, janedoe1.
@@ -285,6 +288,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The name of the city.
@@ -383,6 +389,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The username. For example, janedoe1.
@@ -457,6 +466,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                     - name: name
                       type: keyword
                       description: The username. For example, janedoe1.
@@ -782,6 +794,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The username. For example, janedoe1.
@@ -870,6 +885,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The name of the city.
@@ -967,6 +985,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The username. For example, janedoe1.
@@ -1040,6 +1061,9 @@
                             - name: uid
                               type: keyword
                               description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+                        - name: ldap_person
+                          type: flattened
+                          description: The LDAP attributes of the user.
                         - name: name
                           type: keyword
                           description: The username. For example, janedoe1.
@@ -1270,6 +1294,9 @@
                     - name: uuid
                       type: keyword
                       description: The universally unique identifier of the session.
+                    - name: terminal
+                      type: keyword
+                      description: The Pseudo Terminal associated with the session. Ex, the tty or pts value.
                 - name: terminated_time
                   type: date
                   description: The time when the process was terminated.
@@ -1397,6 +1424,9 @@
                 - name: uuid
                   type: keyword
                   description: The universally unique identifier of the session.
+                - name: terminal
+                  type: keyword
+                  description: The Pseudo Terminal associated with the session. Ex, the tty or pts value.
             - name: terminated_time
               type: date
               description: The time when the process was terminated.

packages/amazon_security_lake/data_stream/discovery/fields/device-fields.yml

@@ -34,6 +34,9 @@
             - name: desc
               type: keyword
               description: The group description.
+            - name: domain
+              type: keyword
+              description: The domain where the group is defined. For example, the LDAP or Active Directory domain.
             - name: name
               type: keyword
               description: The group name.

packages/amazon_security_lake/data_stream/discovery/fields/fields.yml

@@ -126,6 +126,7 @@
       fields:
         - name: data
           type: flattened
+          ignore_malformed: true
           description: The enrichment data associated with the attribute and value. The meaning of this data depends on the type the enrichment record.
         - name: name
           type: keyword

packages/amazon_security_lake/data_stream/event/_dev/deploy/tf/main.tf

@@ -20,36 +20,25 @@ provider "aws" {
   }
 }
 
-# Define a list of file prefixes to be used for creating buckets
-locals {
-  file_prefixes = ["discovery", "findings"]
-}
-
-# Create S3 buckets based on file prefixes
 resource "aws_s3_bucket" "security_lake_logs" {
-  for_each = toset(local.file_prefixes)
-
-  bucket = "security-lake-logs-${each.key}-bucket-${var.TEST_RUN_ID}"
+  bucket = "security-lake-logs-bucket-${var.TEST_RUN_ID}"
 }
 
-# Upload files to corresponding buckets based on their file prefix
+# Upload files to the single bucket with directory structures based on their file prefix
 resource "aws_s3_object" "objects" {
-  for_each = { for file in fileset(var.files_path, "**") : file => file if contains(local.file_prefixes, split("_", file)[0]) }
+  for_each = fileset(var.files_path, "**")
 
-  bucket = aws_s3_bucket.security_lake_logs[split("_", each.value)[0]].id
+  bucket = aws_s3_bucket.security_lake_logs.id
+
+  # Create the directory structure based on the file prefix
+  key    = "${split("_", each.value)[0]}/${each.value}" 
 
-  key    = each.value  # The S3 object key will reflect the nested directory structure
   source = "${var.files_path}/${each.value}"  # Full path to the source file
 
   etag = filemd5("${var.files_path}/${each.value}")
 }
 
-output "bucket_arn_discovery" {
-  value = aws_s3_bucket.security_lake_logs["discovery"].arn
-  description = "The ARN of the 'discovery' bucket"
-}
-
-output "bucket_arn_findings" {
-  value = aws_s3_bucket.security_lake_logs["findings"].arn
-  description = "The ARN of the 'findings' bucket"
-}
+output "bucket_arn" {
+  value = aws_s3_bucket.security_lake_logs.arn
+  description = "The ARN of the S3 bucket"
+}