github: Add latest transforms for issues, dependabot, code_scanning, and secret_scanning alerts

packages/github/changelog.yml

@@ -1,4 +1,27 @@
 # newer versions go on top
+- version: "2.0.0"
+  changes:
+    - description: Update fields inside fingerprint processor in code_scanning, secret_scanning, and dependabot to ingest all event updates.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11518
+    - description: Reformat fields to add package-fields.yml across all datastreams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11518
+    - description: Remove github.state, github.severity due to inconsistency across datastreams. Update dashboards to use corresponding datastream-level fields instead.
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/11518
+    - description: Add latest transforms for github issues, dependabot, code_scanning, and secret_scanning alerts.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11518
+    - description: Add navigation to all dashboards.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11518
+    - description: Upgrade legacy visualization to latest for code_scanning and secret_scanning.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11518
+    - description: Change dashboards to point to destination index for issues, dependabot, code_scanning, and secret_scanning alerts.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11518
 - version: "1.29.3"
   changes:
     - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.

packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml

@@ -21,12 +21,14 @@ processors:
   - fail:
       if: "!(ctx.github.code_scanning instanceof Map)"
       message: Missing JSON object
-  - fingerprint:
-      fields:
-        - github.code_scanning.number
-        - github.code_scanning.updated_at
-      target_field: "_id"
-      ignore_missing: true
+  - append:
+      field: event.type
+      value: creation
+      if: ctx.github?.code_scanning?.fixed_at == null && ctx.github?.code_scanning?.dismissed_at == null
+  - append:
+      field: event.type
+      value: deletion
+      if: ctx.github?.code_scanning?.fixed_at != null || ctx.github?.code_scanning?.dismissed_at != null
   - date:
       field: github.code_scanning.created_at
       formats:
@@ -48,6 +50,13 @@ processors:
       timezone: UTC
       target_field: "@timestamp"
       if: ctx.github.code_scanning.updated_at != null
+  - date:
+      field: github.code_scanning.dismissed_at
+      formats:
+        - ISO8601
+      timezone: UTC
+      target_field: github.code_scanning.dismissed_at
+      if: ctx.github?.code_scanning?.dismissed_at != null
   - rename:
       target_field: _temp
       field: github.code_scanning.repository
@@ -135,24 +144,22 @@ processors:
       target_field: github.code_scanning.number
       if: ctx.github.code_scanning.number == null
       ignore_missing: true
+  - fingerprint:
+      fields:
+        - github.repository.owner.login
+        - github.repository.name
+        - github.code_scanning.number
+        - github.code_scanning.created_at
+        - github.code_scanning.updated_at
+        - github.code_scanning.dismissed_at
+      target_field: "_id"
+      ignore_missing: true
   - lowercase:
       field: github.code_scanning.state
       ignore_missing: true
   - lowercase:
       field: github.code_scanning.rule.security_severity_level
       ignore_missing: true
-  - set:
-      field: github.severity
-      value: "{{{github.code_scanning.rule.security_severity_level}}}"
-      if: ctx.github.code_scanning.rule?.security_severity_level != null
-  - set:
-      field: github.severity
-      value: "undefined"
-      if: ctx.github.severity == null
-  - set:
-      field: github.state
-      value: "{{{github.code_scanning.state}}}"
-      if: ctx.github.code_scanning.state != null
   - rename:
       target_field: _temp.dismissed_by
       field: github.code_scanning.dismissed_by

packages/github/data_stream/code_scanning/fields/agent.yml

@@ -6,18 +6,18 @@
   fields:
     - name: containerized
       type: boolean
-      description: >
-        If the host is a container.
-
+      description: If the host is a container.
     - name: os.build
       type: keyword
       example: "18D109"
-      description: >
-        OS build information.
-
+      description: OS build information.
     - name: os.codename
       type: keyword
       example: "stretch"
-      description: >
-        OS codename, if any.
-
+      description: OS codename, if any.
+- name: input.type
+  type: keyword
+  description: Input Type.
+- name: log.offset
+  type: long
+  description: Log Offset.

packages/github/data_stream/code_scanning/fields/base-fields.yml

@@ -1,23 +1,16 @@
 - name: data_stream.type
-  type: constant_keyword
-  description: Data stream type.
+  external: ecs
 - name: data_stream.dataset
-  type: constant_keyword
-  description: Data stream dataset name.
+  external: ecs
 - name: data_stream.namespace
-  type: constant_keyword
-  description: Data stream namespace.
+  external: ecs
 - name: event.module
   type: constant_keyword
-  description: Event module
   value: github
+  external: ecs
 - name: event.dataset
   type: constant_keyword
-  description: Event dataset
   value: github.code_scanning
-- name: "@timestamp"
-  type: date
-  description: Event timestamp.
-- name: input.type
-  type: keyword
-  description: Type of Filebeat input.
+  external: ecs
+- name: '@timestamp'
+  external: ecs