[Amazon Security Lake] - OCSF v1.1 update with major refactor & adding support for dynamic template and mappings & system tests

packages/amazon_security_lake/data_stream/application_activity/fields/fields.yml

@@ -100,6 +100,9 @@
         - name: zone
           type: keyword
           description: The availability zone in the cloud region, as defined by the cloud provider.
+    - name: command_uid
+      type: keyword
+      description: The command identifier that is associated with this scan event. This ID uniquely identifies the proactive scan command, e.g., if remotely initiated.
     - name: count
       type: long
       description: The number of times that events in the same logical group occurred during the event Start Time to End Time period.
@@ -331,6 +334,33 @@
     - name: message
       type: keyword
       description: The description of the event, as defined by the event source.
+    - name: num_detections
+      type: integer
+      description: The number of detections.
+    - name: num_files
+      type: integer
+      description: The number of files scanned.
+    - name: num_folders
+      type: integer
+      description: The number of folders scanned.
+    - name: num_network_items
+      type: integer
+      description: The number of network items scanned.
+    - name: num_processes
+      type: integer
+      description: The number of processes scanned.
+    - name: num_registry_items
+      type: integer
+      description: The number of registry items scanned.
+    - name: num_resolutions
+      type: integer
+      description: The number of items that were resolved.
+    - name: num_skipped_items
+      type: integer
+      description: The number of items that were skipped.
+    - name: num_trusted_items
+      type: integer
+      description: The number of trusted items.
     - name: observables
       type: group
       fields:
@@ -361,6 +391,42 @@
         - name: value
           type: keyword
           description: The value associated with the observable attribute.
+    - name: policy
+      type: group
+      fields:
+        - name: desc
+          type: keyword
+          description: The description of the policy.
+        - name: group
+          type: group
+          fields:
+            - name: domain
+              type: keyword
+              description: The domain where the group is defined. For example, the LDAP or Active Directory domain.
+            - name: desc
+              type: keyword
+              description: The group description.
+            - name: name
+              type: keyword
+              description: The group name.
+            - name: privileges
+              type: keyword
+              description: The group privileges.
+            - name: type
+              type: keyword
+              description: The type of the group or account.
+            - name: uid
+              type: keyword
+              description: The unique identifier of the group. For example, for Windows events this is the security identifier (SID) of the group.
+        - name: name
+          type: keyword
+          description: 'The policy name. For example: IAM Policy.'
+        - name: uid
+          type: keyword
+          description: A unique identifier of the policy instance.
+        - name: version
+          type: keyword
+          description: The policy version number.
     - name: proxy
       type: group
       fields:
@@ -469,6 +535,25 @@
     - name: raw_data
       type: flattened
       description: The event data as received from the event source.
+    - name: scan
+      type: group
+      description: The Scan object describes characteristics of a proactive scan.
+      fields:
+        - name: name
+          type: keyword
+          description: The administrator-supplied or application-generated name of the scan.
+        - name: type
+          type: keyword
+          description: The type of scan.
+        - name: type_id
+          type: integer
+          description: The type id of the scan.
+        - name: uid
+          type: keyword
+          description: The application-defined unique identifier assigned to an instance of a scan.
+    - name: schedule_uid
+      type: keyword
+      description: The unique identifier of the schedule associated with a scan job.
     - name: severity
       type: keyword
       description: The event severity, normalized to the caption of the severity_id value. In the case of 'Other', it is defined by the event source.
@@ -493,6 +578,9 @@
     - name: status_id
       type: keyword
       description: The normalized identifier of the event status.
+    - name: total
+      type: integer
+      description: The total number of items that were scanned; zero if no items were scanned.
     - name: time
       type: date
       description: The normalized event occurrence time.

packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml

@@ -40,7 +40,7 @@ processors:
   - set:
       field: event.kind
       tag: set_event_kind
-      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1004','1005','1006','1007','3001','3002','3003','3005','3006','4001','4002','4003','4004','4005','4006','4007','4008','4009','4010','4011','4012','4013','5001','5002','5003','5004','5019','6001','6002','6003','6004','6005'].contains(ctx.ocsf.class_uid)
+      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1004','1005','1006','1007','3001','3002','3003','3005','3006','4001','4002','4003','4004','4005','4006','4007','4008','4009','4010','4011','4012','4013','5001','5002','5003','5004','5019','6001','6002','6003','6004','6005','6007'].contains(ctx.ocsf.class_uid)
       value: event
   - set:
       field: event.kind
@@ -136,7 +136,7 @@ processors:
       tag: append_info_into_event_type
       value: info
       allow_duplicates: false
-      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1007','2001', '2002','2003','2004','2005','3001','3002','3003','3005','3006','4001','4002','4003','4004','4005','4006','4007','4008','4009','4010','4011','4012','4013','5001','5002','5003','5004','5019','6002','6003','6004','6005'].contains(ctx.ocsf.class_uid)
+      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1007','2001', '2002','2003','2004','2005','3001','3002','3003','3005','3006','4001','4002','4003','4004','4005','4006','4007','4008','4009','4010','4011','4012','4013','5001','5002','5003','5004','5019','6002','6003','6004','6005','6007'].contains(ctx.ocsf.class_uid)
   - append:
       field: event.type
       tag: append_user_into_event_type
@@ -178,13 +178,13 @@ processors:
       tag: append_start_into_event_type
       value: start
       allow_duplicates: false
-      if: ctx.ocsf?.class_uid != null && ['1007','3002','4001','4013','4007','6002'].contains(ctx.ocsf.class_uid) && ['Launch','Logon','Open','Start','Symmetric Active Exchange','Client Synchronization','Broadcast','Control'].contains(ctx.ocsf.activity_name)
+      if: ctx.ocsf?.class_uid != null && ['1007','3002','4001','4013','4007','6002','6007'].contains(ctx.ocsf.class_uid) && ['Launch','Logon','Open','Start','Started','Symmetric Active Exchange','Client Synchronization','Broadcast','Control'].contains(ctx.ocsf.activity_name)
   - append:
       field: event.type
       tag: append_end_into_event_type
       value: end
       allow_duplicates: false
-      if: ctx.ocsf?.class_uid != null && ['1007','2005','3002','4001','4007','4013','6002'].contains(ctx.ocsf.class_uid) && ['Terminate','Logoff','Close','Stop','Symmetric Passive Response','Server Response','Private Use Case','Other'].contains(ctx.ocsf.activity_name)
+      if: ctx.ocsf?.class_uid != null && ['1007','2005','3002','4001','4007','4013','6002','6007'].contains(ctx.ocsf.class_uid) && ['Terminate','Logoff','Close','Completed','Stop','Symmetric Passive Response','Server Response','Private Use Case','Other'].contains(ctx.ocsf.activity_name)
   - append:
       field: event.type
       tag: append_denied_into_event_type
@@ -220,7 +220,7 @@ processors:
       tag: append_error_into_event_type
       value: error
       allow_duplicates: false
-      if: ctx.ocsf?.class_uid != null && ['6004'].contains(ctx.ocsf.class_uid) && ['Access Error'].contains(ctx.ocsf.activity_name)
+      if: ctx.ocsf?.class_uid != null && ['6004','6007'].contains(ctx.ocsf.class_uid) && ['Access Error','Error'].contains(ctx.ocsf.activity_name)
   - set:
       field: cloud.account.id
       tag: set_cloud_account_uid
@@ -705,7 +705,7 @@ processors:
             ignore_missing: true
   - pipeline:
       name: '{{ IngestPipeline "pipeline_object_actor" }}'
-      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1004','1005','1006','1007','2002','2003','3001','3002','3003','3004','3005','4001','4002','4003','4004','4005','4006','4007','4008','4009','4010','4011','4012','4013','5001','5002','5003','5019','6001','6002','6003','6004','6005'].contains(ctx.ocsf.class_uid) && ctx.ocsf.actor != null
+      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1004','1005','1006','1007','2002','2003','3001','3002','3003','3004','3005','4001','4002','4003','4004','4005','4006','4007','4008','4009','4010','4011','4012','4013','5001','5002','5003','5019','6001','6002','6003','6004','6005','6007'].contains(ctx.ocsf.class_uid) && ctx.ocsf.actor != null
       tag: pipeline_object_actor
       ignore_missing_pipeline: true
   - pipeline:
@@ -720,7 +720,7 @@ processors:
       ignore_missing_pipeline: true
   - pipeline:
       name: '{{ IngestPipeline "pipeline_object_device" }}'
-      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1004','1005','1006','1007','2002','2003','3001','3002','3003','3004','3005','4001','4002','4003','4004','4005','4006','4007','4008','4009','4011','4012','4013','5001','5002','5004','5019','6001','6002','6004'].contains(ctx.ocsf.class_uid) && ctx.ocsf.device != null
+      if: ctx.ocsf?.class_uid != null && ['1001','1002','1003','1004','1005','1006','1007','2002','2003','3001','3002','3003','3004','3005','4001','4002','4003','4004','4005','4006','4007','4008','4009','4011','4012','4013','5001','5002','5004','5019','6001','6002','6004','6007'].contains(ctx.ocsf.class_uid) && ctx.ocsf.device != null
       tag: pipeline_object_device
       ignore_missing_pipeline: true
   - pipeline:

packages/amazon_security_lake/data_stream/event/fields/misc-fields.yml

@@ -28,7 +28,7 @@
     - name: verdict_id
       type: integer
       description: The normalized verdict of an Incident.
-      # These fields are used to store misc information about a discovery category event. 
+    # These fields are used to store misc information about a discovery category event. 
     - name: prev_security_states
       type: group
       description: The previous security states of the device.
@@ -55,3 +55,59 @@
         - name: state_id
           type: integer
           description: The security state of the managed entity.
+    # These fields are used to store misc information about an application activity category event.
+    - name: command_uid
+      type: keyword
+      description: The command identifier that is associated with this scan event. This ID uniquely identifies the proactive scan command, e.g., if remotely initiated.
+    - name: num_detections
+      type: integer
+      description: The number of detections.
+    - name: num_files
+      type: integer
+      description: The number of files scanned.
+    - name: num_folders
+      type: integer
+      description: The number of folders scanned.
+    - name: num_network_items
+      type: integer
+      description: The number of network items scanned.
+    - name: num_processes
+      type: integer
+      description: The number of processes scanned.
+    - name: num_registry_items
+      type: integer
+      description: The number of registry items scanned.
+    - name: num_resolutions
+      type: integer
+      description: The number of items that were resolved.
+    - name: num_skipped_items
+      type: integer
+      description: The number of items that were skipped.
+    - name: num_trusted_items
+      type: integer
+      description: The number of trusted items.
+    - name: policy
+      type: flattened
+      description: The policy that was used to scan the device.
+    - name: scan
+      type: group
+      description: The Scan object describes characteristics of a proactive scan.
+      fields:
+        - name: name
+          type: keyword
+          description: The administrator-supplied or application-generated name of the scan.
+        - name: type
+          type: keyword
+          description: The type of scan.
+        - name: type_id
+          type: integer
+          description: The type id of the scan.
+        - name: uid
+          type: keyword
+          description: The application-defined unique identifier assigned to an instance of a scan.
+    - name: schedule_uid
+      type: keyword
+      description: The unique identifier of the schedule associated with a scan job.
+    - name: total
+      type: integer
+      description: The total number of items that were scanned; zero if no items were scanned.

packages/amazon_security_lake/docs/README.md

@@ -704,6 +704,7 @@ This is the `Event` dataset.
 | ocsf.codes | The list of return codes to the FTP command. | long |
 | ocsf.command | The command name. | keyword |
 | ocsf.command_responses | The list of responses to the FTP command. | keyword |
+| ocsf.command_uid | The command identifier that is associated with this scan event. This ID uniquely identifies the proactive scan command, e.g., if remotely initiated. | keyword |
 | ocsf.comment | The user provided comment about why the entity was changed. | keyword |
 | ocsf.compliance.control | A Control is prescriptive, prioritized, and simplified set of best practices that one can use to strengthen their cybersecurity posture. e.g. AWS SecurityHub Controls, CIS Controls. | keyword |
 | ocsf.compliance.requirements | A list of requirements associated to a specific control in an industry or regulatory framework. e.g. NIST.800-53.r5 AU-10. | keyword |
@@ -1642,6 +1643,15 @@ This is the `Event` dataset.
 | ocsf.module.type | The module type. | keyword |
 | ocsf.name | The name of the data affiliated with the command. | keyword |
 | ocsf.nist | The NIST Cybersecurity Framework recommendations for managing the cybersecurity risk. | keyword |
+| ocsf.num_detections | The number of detections. | integer |
+| ocsf.num_files | The number of files scanned. | integer |
+| ocsf.num_folders | The number of folders scanned. | integer |
+| ocsf.num_network_items | The number of network items scanned. | integer |
+| ocsf.num_processes | The number of processes scanned. | integer |
+| ocsf.num_registry_items | The number of registry items scanned. | integer |
+| ocsf.num_resolutions | The number of items that were resolved. | integer |
+| ocsf.num_skipped_items | The number of items that were skipped. | integer |
+| ocsf.num_trusted_items | The number of trusted items. | integer |
 | ocsf.observables.name | The full name of the observable attribute. The name is a pointer/reference to an attribute within the event data. For example: file.name. | keyword |
 | ocsf.observables.reputation.base_score | The reputation score as reported by the event source. | double |
 | ocsf.observables.reputation.provider | The provider of the reputation information. | keyword |
@@ -1651,6 +1661,7 @@ This is the `Event` dataset.
 | ocsf.observables.type_id | The observable value type identifier. | keyword |
 | ocsf.observables.value | The value associated with the observable attribute. | keyword |
 | ocsf.open_type | Indicates how the file was opened (e.g. normal, delete on close). | keyword |
+| ocsf.policy | The policy that was used to scan the device. | flattened |
 | ocsf.port | The dynamic port established for impending data transfers. | long |
 | ocsf.precision | The NTP precision quantifies a clock's accuracy and stability in log2 seconds, as defined in RFC-5905. | integer |
 | ocsf.prev_security_states.state | The security state, normalized to the caption of the state_id value. | keyword |
@@ -1806,6 +1817,11 @@ This is the `Event` dataset.
 | ocsf.risk_level | The risk level, normalized to the caption of the risk_level_id value. In the case of 'Other', it is defined by the event source. | keyword |
 | ocsf.risk_level_id | The normalized risk level id. | keyword |
 | ocsf.risk_score | The risk score as reported by the event source. | long |
+| ocsf.scan.name | The administrator-supplied or application-generated name of the scan. | keyword |
+| ocsf.scan.type | The type of scan. | keyword |
+| ocsf.scan.type_id | The type id of the scan. | integer |
+| ocsf.scan.uid | The application-defined unique identifier assigned to an instance of a scan. | keyword |
+| ocsf.schedule_uid | The unique identifier of the schedule associated with a scan job. | keyword |
 | ocsf.security_level | The current security level of the entity. | keyword |
 | ocsf.security_level_id | The current security level of the entity. | integer |
 | ocsf.security_states.state | The security state, normalized to the caption of the state_id value. | keyword |
@@ -1914,6 +1930,7 @@ This is the `Event` dataset.
 | ocsf.tls.server_ciphers | The server cipher suites that were exchanged during the TLS handshake negotiation. | keyword |
 | ocsf.tls.sni | The Server Name Indication (SNI) extension sent by the client. | keyword |
 | ocsf.tls.version | The TLS protocol version. | keyword |
+| ocsf.total | The total number of items that were scanned; zero if no items were scanned. | integer |
 | ocsf.traffic.bytes | The total number of bytes (in and out). | long |
 | ocsf.traffic.bytes_in | The number of bytes sent from the destination to the source. | long |
 | ocsf.traffic.bytes_out | The number of bytes sent from the source to the destination. | long |