Merge branch 'main' of github.com:elastic/elastic-agent into runtime-…

…manager-sub
elastic · Jul 24, 2023 · 38dbf9d · 38dbf9d
2 parents de94bcf + 2a3dfaa
commit 38dbf9d
Show file tree

Hide file tree

Showing 99 changed files with 2,894 additions and 610 deletions.
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -114,7 +114,7 @@ pipeline {
               withGithubNotify(context: "Test-${PLATFORM}") {
                 withMageEnv(){
                   dir("${BASE_DIR}"){
-                    withEnv(["TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
+                    withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
                       cmd(label: 'Go unitTest', script: 'mage unitTest')
                     }
                   }
@@ -330,7 +330,7 @@ pipeline {
               withGithubNotify(context: "Test-${PLATFORM}") {
                 withMageEnv(){
                   dir("${BASE_DIR}"){
-                    withEnv(["TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
+                    withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
                       cmd(label: 'Go unitTest', script: 'mage unitTest')
                     }
                   }
@@ -380,7 +380,7 @@ pipeline {
             withGithubNotify(context: "Test-darwin-aarch64") {
               withMageEnv(){
                 dir("${BASE_DIR}"){
-                  withEnv(["TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
+                  withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
                     cmd(label: 'Go unitTest', script: 'mage unitTest')
                   }
                 }

diff --git a/changelog/fragments/1688069371-Tamper-protected-Endpoint-uninstall.yaml b/changelog/fragments/1688069371-Tamper-protected-Endpoint-uninstall.yaml
@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Tamper protected Endpoint uninstall
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Add new `--uninstall-token` to allow uninstall when Endpoint protection is enabled.
+  Enable unenroll and upgrade actions to complete successfully when Endpoint protection is enabled.
+  Enable Endpoint integration removal when Endpoint protection is enabled.
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/2781
diff --git a/...elog/fragments/1689328899-Elastic-Agent-container-runs-on-Azure-Container-Instances-.yaml b/...elog/fragments/1689328899-Elastic-Agent-container-runs-on-Azure-Container-Instances-.yaml
@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Elastic-Agent container runs on Azure Container Instances 
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+#description: 
+
+# Affected component; a word indicating the component this changeset affects.
+component: elastic-agent
+
+# PR number; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: 3084
+
+# Issue number; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: 82
diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/activemq.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/activemq.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: filestream-activemq
+      id: filestream-activemq-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:
@@ -45,6 +46,7 @@ inputs:
             - activemq-log
       data_stream.namespace: default
     - name: activemq/metrics-activemq
+      id: activemq/metrics-activemq-${kubernetes.hints.container_id}
       type: activemq/metrics
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: filestream-apache
+      id: filestream-apache-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:
@@ -41,6 +42,7 @@ inputs:
             - apache-error
       data_stream.namespace: default
     - name: httpjson-apache
+      id: httpjson-apache-${kubernetes.hints.container_id}
       type: httpjson
       use_output: default
       streams:
@@ -122,6 +124,7 @@ inputs:
             - apache-error
       data_stream.namespace: default
     - name: apache/metrics-apache
+      id: apache/metrics-apache-${kubernetes.hints.container_id}
       type: apache/metrics
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cassandra.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cassandra.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: filestream-cassandra
+      id: filestream-cassandra-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:
@@ -27,6 +28,7 @@ inputs:
             - cassandra-systemlogs
       data_stream.namespace: default
     - name: jolokia/metrics-cassandra
+      id: jolokia/metrics-cassandra-${kubernetes.hints.container_id}
       type: jolokia/metrics
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml
@@ -1,43 +1,35 @@
 inputs:
-    - name: filestream-cef
-      type: filestream
+    - name: udp-cef
+      id: udp-cef-${kubernetes.hints.container_id}
+      type: udp
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true
           data_stream:
             dataset: cef.log
             type: logs
-          exclude_files:
-            - .gz$
-          parsers:
-            - container:
-                format: auto
-                stream: ${kubernetes.hints.cef.log.stream|'all'}
-          paths:
-            - /var/log/containers/*${kubernetes.hints.container_id}.log
+          host: localhost:9003
           processors:
             - rename:
                 fields:
                     - from: message
                       to: event.original
             - decode_cef:
                 field: event.original
-          prospector:
-            scanner:
-                symlinks: true
           tags:
             - cef
             - forwarded
       data_stream.namespace: default
-    - name: udp-cef
-      type: udp
+    - name: tcp-cef
+      id: tcp-cef-${kubernetes.hints.container_id}
+      type: tcp
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true
           data_stream:
             dataset: cef.log
             type: logs
-          host: localhost:9003
+          host: localhost:9004
           processors:
             - rename:
                 fields:
@@ -49,22 +41,33 @@ inputs:
             - cef
             - forwarded
       data_stream.namespace: default
-    - name: tcp-cef
-      type: tcp
+    - name: filestream-cef
+      id: filestream-cef-${kubernetes.hints.container_id}
+      type: filestream
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true
           data_stream:
             dataset: cef.log
             type: logs
-          host: localhost:9004
+          exclude_files:
+            - .gz$
+          parsers:
+            - container:
+                format: auto
+                stream: ${kubernetes.hints.cef.log.stream|'all'}
+          paths:
+            - /var/log/containers/*${kubernetes.hints.container_id}.log
           processors:
             - rename:
                 fields:
                     - from: message
                       to: event.original
             - decode_cef:
                 field: event.original
+          prospector:
+            scanner:
+                symlinks: true
           tags:
             - cef
             - forwarded

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/checkpoint.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/checkpoint.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: filestream-checkpoint
+      id: filestream-checkpoint-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:
@@ -34,6 +35,7 @@ inputs:
             - forwarded
       data_stream.namespace: default
     - name: tcp-checkpoint
+      id: tcp-checkpoint-${kubernetes.hints.container_id}
       type: tcp
       use_output: default
       streams:
@@ -48,6 +50,7 @@ inputs:
             - forwarded
       data_stream.namespace: default
     - name: udp-checkpoint
+      id: udp-checkpoint-${kubernetes.hints.container_id}
       type: udp
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cockroachdb.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cockroachdb.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: prometheus/metrics-cockroachdb
+      id: prometheus/metrics-cockroachdb-${kubernetes.hints.container_id}
       type: prometheus/metrics
       use_output: default
       streams:
@@ -22,6 +23,7 @@ inputs:
           username: ${kubernetes.hints.cockroachdb.status.username|kubernetes.hints.cockroachdb.username|''}
       data_stream.namespace: default
     - name: filestream-cockroachdb
+      id: filestream-cockroachdb-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml
@@ -1,35 +1,6 @@
 inputs:
-    - name: aws-s3-crowdstrike
-      type: aws-s3
-      use_output: default
-      streams:
-        - condition: ${kubernetes.hints.crowdstrike.fdr.enabled} == true or ${kubernetes.hints.crowdstrike.enabled} == true
-          data_stream:
-            dataset: crowdstrike.fdr
-            type: logs
-          queue_url: null
-          sqs.notification_parsing_script.source: |
-            function parse(n) {
-              var m = JSON.parse(n);
-              var evts = [];
-              var files = m.files;
-              var bucket = m.bucket;
-              if (!Array.isArray(files) || (files.length == 0) || bucket == null || bucket == "") {
-                return evts;
-              }
-              files.forEach(function(f){
-                var evt = new S3EventV2();
-                evt.SetS3BucketName(bucket);
-                evt.SetS3ObjectKey(f.path);
-                evts.push(evt);
-              });
-              return evts;
-            }
-          tags:
-            - forwarded
-            - crowdstrike-fdr
-      data_stream.namespace: default
     - name: filestream-crowdstrike
+      id: filestream-crowdstrike-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:
@@ -77,3 +48,34 @@ inputs:
             - forwarded
             - crowdstrike-fdr
       data_stream.namespace: default
+    - name: aws-s3-crowdstrike
+      id: aws-s3-crowdstrike-${kubernetes.hints.container_id}
+      type: aws-s3
+      use_output: default
+      streams:
+        - condition: ${kubernetes.hints.crowdstrike.fdr.enabled} == true or ${kubernetes.hints.crowdstrike.enabled} == true
+          data_stream:
+            dataset: crowdstrike.fdr
+            type: logs
+          queue_url: null
+          sqs.notification_parsing_script.source: |
+            function parse(n) {
+              var m = JSON.parse(n);
+              var evts = [];
+              var files = m.files;
+              var bucket = m.bucket;
+              if (!Array.isArray(files) || (files.length == 0) || bucket == null || bucket == "") {
+                return evts;
+              }
+              files.forEach(function(f){
+                var evt = new S3EventV2();
+                evt.SetS3BucketName(bucket);
+                evt.SetS3ObjectKey(f.path);
+                evts.push(evt);
+              });
+              return evts;
+            }
+          tags:
+            - forwarded
+            - crowdstrike-fdr
+      data_stream.namespace: default
diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cyberarkpas.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cyberarkpas.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: tcp-cyberarkpas
+      id: tcp-cyberarkpas-${kubernetes.hints.container_id}
       type: tcp
       use_output: default
       streams:
@@ -16,6 +17,7 @@ inputs:
           tcp: null
       data_stream.namespace: default
     - name: udp-cyberarkpas
+      id: udp-cyberarkpas-${kubernetes.hints.container_id}
       type: udp
       use_output: default
       streams:
@@ -32,6 +34,7 @@ inputs:
           udp: null
       data_stream.namespace: default
     - name: filestream-cyberarkpas
+      id: filestream-cyberarkpas-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: filestream-elasticsearch
+      id: filestream-elasticsearch-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams:
@@ -124,6 +125,7 @@ inputs:
                 symlinks: true
       data_stream.namespace: default
     - name: elasticsearch/metrics-elasticsearch
+      id: elasticsearch/metrics-elasticsearch-${kubernetes.hints.container_id}
       type: elasticsearch/metrics
       use_output: default
       streams:

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/endpoint.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/endpoint.yml
@@ -1,5 +1,6 @@
 inputs:
     - name: filestream-endpoint
+      id: filestream-endpoint-${kubernetes.hints.container_id}
       type: filestream
       use_output: default
       streams: