Merge branch 'main' into patch-1

.github/CODEOWNERS

@@ -12,6 +12,12 @@
 # https://github.community/t/codeowners-file-with-a-not-file-type-condition/1423/9
 CHANGELOG*
 
+# The tech leads of the teams working in Beats share ownership of the Go module dependencies and related files.
+/.github/CODEOWNERS/ @elastic/beats-tech-leads
+/.go.mod/ @elastic/beats-tech-leads
+/.go.sum/ @elastic/beats-tech-leads
+/NOTICE.txt/ @elastic/beats-tech-leads
+
 /.ci/ @elastic/elastic-agent-data-plane
 /.github/ @elastic/elastic-agent-data-plane
 /auditbeat/ @elastic/security-external-integrations
@@ -103,6 +109,7 @@ CHANGELOG*
 /x-pack/filebeat/input/azureblobstorage/ @elastic/security-external-integrations
 /x-pack/filebeat/input/azureeventhub/ @elastic/obs-cloud-monitoring
 /x-pack/filebeat/input/cel/ @elastic/security-external-integrations
+/x-pack/filebeat/input/cometd/ @elastic/obs-infraobs-integrations
 /x-pack/filebeat/input/entityanalytics/ @elastic/security-external-integrations
 /x-pack/filebeat/input/gcppubsub/ @elastic/security-external-integrations
 /x-pack/filebeat/input/gcs/ @elastic/security-external-integrations

.github/dependabot.yml

@@ -1,15 +1,64 @@
 ---
 version: 2
+# This section is segmented by the responsible GitHub teams in order
+# to make it clear who is responsible for reviewing.
 updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
+  - package-ecosystem: gomod
+    directory: /
     schedule:
-      interval: "daily"
+      interval: daily
     labels:
       - automation
+      - dependabot
       - Team:Elastic-Agent-Data-Plane
     allow:
-      - dependency-name: "github.com/elastic/*"
+      - dependency-name: github.com/elastic/elastic-agent-autodiscover
+      - dependency-name: github.com/elastic/elastic-agent-client/*
+      - dependency-name: github.com/elastic/elastic-agent-libs
+      - dependency-name: github.com/elastic/elastic-agent-shipper-client
+      - dependency-name: github.com/elastic/elastic-agent-system-metrics
+      - dependency-name: github.com/elastic/go-concert
+      - dependency-name: github.com/elastic/go-elasticsearch/*
+      - dependency-name: github.com/elastic/go-licenser
+      - dependency-name: github.com/elastic/go-lookslike
+      - dependency-name: github.com/elastic/go-lumber
+      - dependency-name: github.com/elastic/go-structform
+      - dependency-name: github.com/elastic/go-sysinfo
+      - dependency-name: github.com/elastic/go-ucfg
+      - dependency-name: github.com/elastic/gosigar
+      - dependency-name: go.elastic.co/apm/*
+      - dependency-name: go.elastic.co/ecszap
+      - dependency-name: go.elastic.co/go-licence-detector
     reviewers:
-      - "elastic/elastic-agent-data-plane"
-    open-pull-requests-limit: 10
+      - elastic/elastic-agent-data-plane
+    open-pull-requests-limit: 2
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: daily
+    labels:
+      - automation
+      - dependabot
+      - Team:Security-External Integrations
+    allow:
+      # Skip github.com/elastic/mito because it requires documentation updates.
+      - dependency-name: github.com/elastic/go-libaudit/*
+      - dependency-name: github.com/elastic/go-perf
+      - dependency-name: github.com/elastic/go-seccomp-bpf
+      - dependency-name: github.com/elastic/toutoumomoma
+    reviewers:
+      - elastic/security-external-integrations
+    open-pull-requests-limit: 2
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: daily
+    labels:
+      - automation
+      - dependabot
+      - Team:Service-Integrations
+    allow:
+      - dependency-name: github.com/elastic/bayeux
+    reviewers:
+      - elastic/obs-infraobs-integrations
+    open-pull-requests-limit: 2

CHANGELOG-developer.next.asciidoc

@@ -81,6 +81,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Fix the integration testcase docker port mapping for sql and oracle modules {pull}34221[34221]
 - Fix the ingest pipeline for mysql slowlog to parse schema name with dash {pull}34371[34372]
 - Fix the multiple host support for mongodb module {pull}34624[34624]
+- Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138]
 
 ==== Added
 
@@ -144,6 +145,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Add support for `credentials_json` in `gcp` module, all metricsets {pull}29584[29584]
 - Add gcp firestore metricset. {pull}29918[29918]
 - Added TESTING_FILEBEAT_FILEPATTERN option for filebeat module pytests {pull}30103[30103]
+- Improve tests files with shorter statements. {pull}35667[35667]
 - Add gcp dataproc metricset. {pull}30008[30008]
 - Add Github action for linting
 - Add regex support for drop_fields processor.
@@ -154,6 +156,10 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Add the file path of the instance lock on the error when it's is already locked {pull}33788[33788]
 - Add DropFields processor to js API {pull}33458[33458]
 - Add support for different folders when testing data {pull}34467[34467]
+- Add logging of metric registration in inputmon. {pull}35647[35647]
+- Add Okta API package for entity analytics. {pull}35478[35478]
+- Add benchmarking to HTTPJSON input testing. {pull}35138[35138]
+- Allow non-AWS endpoints for testing Filebeat awss3 input. {issue}35496[35496] {pull}35520[35520]
 
 ==== Deprecated
 

CHANGELOG.asciidoc

@@ -3,6 +3,124 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.8.1]]
+=== Beats version 8.8.1
+https://github.com/elastic/beats/compare/v8.8.0\...v8.8.1[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- 'add_cloud_metadata' processor: add `cloud.region` field for GCE cloud provider
+- 'add_cloud_metadata' processor: update Azure metadata API version to get missing `cloud.account.id` field
+
+*Filebeat*
+
+- Fix "Can only start an input when all related states are finished" error when running under Elastic Agent {pull}35250[35250] {issue}33653[33653]
+- [system] Sync system/auth dataset with system integration 1.29.0. {pull}35581[35581]
+- Fix filestream false positive log error "filestream input with ID 'xyz' already exists" {issue}31767[31767]
+- Fix error when trying to use `include_message` parser {issue}35440[35440]
+
+==== Added
+
+*Filebeat*
+
+- Add sanitization capabilities to azure-eventhub input {pull}34874[34874]
+
+*Auditbeat*
+- Migration of system/package module storage from gob encoding to flatbuffer encoding in bolt db. {pull}34817[34817]
+
+*Metricbeat*
+
+- Support collecting metrics from both the monitoring account and linked accounts from AWS CloudWatch. {pull}35540[35540]
+- Add new parameter `include_linked_accounts` to enable/disable metrics collection from multiple linked AWS Accounts {pull}35648[35648]
+
+
+[[release-notes-8.8.0]]
+=== Beats version 8.8.0
+https://github.com/elastic/beats/compare/v8.7.1...v8.8.0[View commits]
+
+
+==== Bugfixes
+
+*Affecting all Beats*
+- Fix race condition when stopping runners {pull}32433[32433]
+- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491]
+- The Elasticsearch output now splits large requests instead of dropping them when it receives a StatusRequestEntityTooLarge error. {pull}34911[34911]
+- In cases where the matcher detects a non-string type in a match statement, report the error as a debug statement, and not a warning statement. {pull}35119[35119]
+- `add_cloud_metadata` processor: Add `cloud.region` field for GCE cloud provider.
+- `add_cloud_metadata` processor: Update Azure metadata API version to get missing `cloud.account.id` field.
+
+*Filebeat*
+- [GCS Input] Added missing locks for safe concurrency. {pull}34914[34914]
+- Fix the `ignore_inactive` option being ignored in Filebeat's filestream input. {pull}34770[34770]
+- Add input instance ID to request trace filename for httpjson and cel inputs. {pull}35024[35024]
+- Sanitize filenames for request tracer in httpjson input. {pull}35143[35143]
+- Sanitize filenames for request tracer in cel input. {pull}35154[35154]
+- Fix the grok expression outputs of log files. {pull}35221[35221]
+- Move repeated Windows event channel not found errors in winlog input to debug level.  {issue}35314[35314] {pull}35317[35317]
+- Fix crash when processing forwarded logs missing a message. {issue}34705[34705] {pull}34865[34865]
+- Fix crash when loading azurewebstorage cursor with no partially processed data. {pull}35433[35433]
+
+*Heartbeat*
+
+- Fix panics when parsing when HTTP URL is not parseable. {pull}34702[34702]
+- Fix broken state ID location naming. {pull}35336[35336]
+- Fix project monitor temp directories permission to include group access. {pull}35398[35398]
+- Fix output pipeline exit on `run_once`. {pull}35376[35376]
+- Fix formatting issue with socket trace timeout. {pull}35434[35434]
+
+*Metricbeat*
+
+- Make generic SQL GA. {pull}34637[34637]
+- Collect missing `remote_cluster` in Elasticsearch CCR metricset. {pull}34957[34957]
+- Add context with timeout in AWS API calls. {pull}35425[35425]
+
+*Osquerybeat*
+
+- Adds the `elastic_file_analysis` table to the Osquery extension for macOS builds. {pull}35056[35056]
+
+*Packetbeat*
+
+- Fix BPF filter setting not being applied to sniffers. {issue}35363[35363] {pull}35484[35484]
+
+*Winlogbeat*
+
+- Move repeated channel not found errors to debug level.  {issue}35314[35314] {pull}35317[35317]
+- Fix panic due to misrepresented buffer use. {pull}35437[35437]
+- Allow program termination when attempting to open an absent channel. {pull}35474[35474]
+
+==== Added
+
+*Filebeat*
+
+- Add metric `sqs_messages_waiting_gauge` for aws-s3 input. {pull}34488[34488]
+- Add support for Okta debug attributes, `risk_reasons`, `risk_behaviors` and `factor`. {issue}33677[33677] {pull}34508[34508]
+- Add `nginx.ingress_controller.upstream.ip` to `related.ip` {issue}34645[34645] {pull}34672[34672]
+- Include NAT and firewall IPs in `related.ip` in Fortinet Firewall module. {issue}34640[34640] {pull}34673[34673]
+- Add UNIX socket log parsing for NGINX `ingress_controller`. {pull}34732[34732]
+- Add metric `sqs_worker_utilization` for aws-s3 input. {pull}34793[34793]
+- Register MIME handlers for CSV types in CEL input. {pull}34934[34934]
+- Add MySQL authentication message parsing and `related.ip` and `related.user` fields. {pull}34810[34810]
+- Mention `mito` CEL tool in CEL input docs. {pull}34959[34959]
+- Add nginx ingress_controller parsing if one of upstreams fails to return response. {pull}34787[34787]
+- Allow neflow v9 and ipfix templates to be shared between source addresses. {pull}35036[35036]
+- Add support for collecting IPv6 metrics. {pull}35123[35123]
+- Add Oracle authentication messages parsing {pull}35127[35127]
+
+*Heartbeat*
+- Add status to monitor run log report.
+- Remov Beta label for browser monitors. {pull}35424[35424].
+
+*Metricbeat*
+
+- Add GCP Carbon Footprint metricbeat data. {pull}34820[34820]
+- Add event loop utilization metric to Kibana module. {pull}35020[35020]
+
+*Winlogbeat*
+
+- Add `event.category` and `event.type` to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255. {pull}35193[35193]
+
 [[release-notes-8.7.1]]
 === Beats version 8.7.1
 https://github.com/elastic/beats/compare/v8.7.0\...v8.7.1[View commits]
@@ -123,6 +241,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Metrics hosted by the HTTP monitoring endpoint for the `aws-cloudwatch`, `aws-s3`, `cel`, and `lumberjack` inputs are now available under `/inputs/` instead of `/dataset`.
 
 *Heartbeat*
+
 - Users can now configure max scheduler job limits per monitor type via env var. {pull}34307[34307]
 
 - Remove host and port matching restrictions on hint-generated monitors. {pull}34376[34376]
@@ -153,6 +272,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 === Beats version 8.6.2
 https://github.com/elastic/beats/compare/v8.6.1\...v8.6.2[View commits]
 
+
 ==== Bugfixes
 
 *Affecting all Beats*
@@ -199,10 +319,12 @@ https://github.com/elastic/beats/compare/v8.6.1\...v8.6.2[View commits]
 
 ==== Added
 
+
 *Filebeat*
 
 - Added support for HTTP destination override to Google Cloud Storage input. {pull}34413[34413]
 
+
 [[release-notes-8.6.1]]
 === Beats version 8.6.1
 https://github.com/elastic/beats/compare/v8.6.0\...v8.6.1[View commits]
@@ -9537,7 +9659,7 @@ https://github.com/elastic/beats/compare/1.0.0-beta3\...1.0.0-beta4[Check
 - Add tls configuration support to elasticsearch and logstash outputers #139
 - All external dependencies were updated to the latest version. Update to Golang 1.5.1 #162
 - Guarantee ES index is based in UTC time zone #164
-- Cache: optional per element timeout #144
+- Cache: optional per element timeout #144
 - Make it possible to set hosts in different ways. #135
 - Expose more TLS config options #124
 - Use the Beat name in the default configuration file path #99
@@ -9577,3 +9699,4 @@ https://github.com/elastic/beats/compare/1.0.0-beta3\...1.0.0-beta4[Check
 - Redis output was deprecated #169 #145
 - Host and port configuration options are deprecated. They are replaced by the hosts
  configuration option. #141
+

CHANGELOG.next.asciidoc

@@ -4,7 +4,7 @@
 :pull: https://github.com/elastic/beats/pull/
 
 === Beats version HEAD
-https://github.com/elastic/beats/compare/v8.7.1\...main[Check the HEAD diff]
+https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 ==== Breaking changes
 
@@ -122,10 +122,14 @@ https://github.com/elastic/beats/compare/v8.7.1\...main[Check the HEAD diff]
 - Move repeated Windows event channel not found errors in winlog input to debug level.  {issue}35314[35314] {pull}35317[35317]
 - Fix crash when processing forwarded logs missing a message. {issue}34705[34705] {pull}34865[34865]
 - Fix crash when loading azurewebstorage cursor with no partially processed data. {pull}35433[35433]
+- Add support in s3 input for JSON with array of objects. {pull}35475[35475]
 - RFC5424 syslog timestamps with offset 'Z' will be treated as UTC rather than using the default timezone. {pull}35360[35360]
 - [system] sync system/auth dataset with system integration 1.29.0. {pull}35581[35581]
 - [GCS Input] - Fixed an issue where bucket_timeout was being applied to the entire bucket poll interval and not individual bucket object read operations. Fixed a map write concurrency issue arising from data races when using a high number of workers. Fixed the flaky tests that were present in the GCS test suit. {pull}35605[35605]
 - Fix filestream false positive log error "filestream input with ID 'xyz' already exists" {issue}31767[31767]
+- Fix error when trying to use `include_message` parser {issue}35440[35440]
+- Fix handling of IPv6 unspecified addresses in TCP input. {issue}35064[35064] {pull}35637[35637]
+- Fixed a minor code error in the GCS input scheduler where a config value was being used directly instead of the source struct. {pull}35729[35729]
 
 *Heartbeat*
 
@@ -147,6 +151,8 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Fix project monitor temp directories permission to include group access. {pull}35398[35398]
 - Fix output pipeline exit on run_once. {pull}35376[35376]
 - Fix formatting issue with socket trace timeout. {pull}35434[35434]
+- Update gval version. {pull}35636[35636]
+- Fix serialization of processors when running diagnostics. {pull}35698[35698]
 
 *Heartbeat*
 
@@ -231,6 +237,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Reload Beat when TLS certificates or key files are modified. {issue}34408[34408] {pull}34416[34416]
 - Upgrade version of elastic-agent-autodiscover to v0.6.1 for improved memory consumption on k8s. {pull}35483[35483]
 - Added `orchestrator.cluster.id` and `orchestrator.cluster.name` fields to the add_cloud_metadata processor, AWS cloud provider. {pull}35182[35182]
+- Lowercase reported hostnames per Elastic Common Schema (ECS) guidelines for the host.name field. Upgraded github.com/elastic/go-sysinfo to 1.11.0. {pull}35652[35652]
 
 *Auditbeat*
 
@@ -298,6 +305,9 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Add support for collecting `httpjson` metrics. {pull}35392[35392]
 - Add XML decoding support to CEL. {issue}34438[34438] {pull}35372[35372]
 - Mark CEL input as GA. {pull}35559[35559]
+- Add metrics for gcp-pubsub input. {pull}35614[35614]
+- [GCS] Added scheduler debug logs and improved the context passing mechanism by removing them from struct params and passing them as function arguments. {pull}35674[35674]
+- Allow non-AWS endpoints for awss3 input. {issue}35496[35496] {pull}35520[35520]
 
 *Auditbeat*
    - Migration of system/package module storage from gob encoding to flatbuffer encoding in bolt db. {pull}34817[34817]
@@ -335,6 +345,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 *Packetbeat*
 
 - Added `packetbeat.interfaces.fanout_group` to allow a Packetbeat sniffer to join an AF_PACKET fanout group. {issue}35451[35451] {pull}35453[35453]
+- Add AF_PACKET metrics. {issue}35428[35428] {pull}35489[35489]
 
 *Winlogbeat*