Merge branch 'main' into 1127/service-refactor-request

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
+            BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -322,8 +322,8 @@ steps:
         env:
           BWC_VERSION: 7.16.3
 
-      - label: "{{matrix.image}} / 7.17.24 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.24
+      - label: "{{matrix.image}} / 7.17.25 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.25
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -337,7 +337,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.24
+          BWC_VERSION: 7.17.25
 
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1

.buildkite/pipelines/periodic.yml

@@ -342,8 +342,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 7.17.24 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.24#bwcTest
+      - label: 7.17.25 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.25#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -353,7 +353,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.24
+          BWC_VERSION: 7.17.25
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
+            BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
+            BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -16,7 +16,7 @@ BWC_VERSION:
   - "7.14.2"
   - "7.15.2"
   - "7.16.3"
-  - "7.17.24"
+  - "7.17.25"
   - "8.0.1"
   - "8.1.3"
   - "8.2.3"

.ci/scripts/resolve-dra-manifest.sh

@@ -23,6 +23,12 @@ LATEST_VERSION=$(strip_version $LATEST_BUILD)
 if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
   echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
   NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
+
+  # Temporary
+  if [[ "$ES_VERSION" == "8.16.0" ]]; then
+    NEW_BRANCH="8.x"
+  fi
+
   echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
   LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
 fi

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.24"
+  - "7.17.25"
   - "8.15.2"
   - "8.16.0"

distribution/docker/build.gradle

@@ -515,14 +515,17 @@ subprojects { Project subProject ->
       base = DockerBase.CLOUD_ESS
     } else if (subProject.name.contains('cloud-')) {
       base = DockerBase.CLOUD
+    } else if (subProject.name.contains('wolfi-')) {
+      base = DockerBase.WOLFI
     }
 
     final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
     final String extension = base == DockerBase.UBI ? 'ubi.tar' :
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
         (base == DockerBase.CLOUD ? 'cloud.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
-            'docker.tar')))
+            (base == DockerBase.WOLFI ? 'wolfi.tar' :
+              'docker.tar'))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

docs/changelog/109193.yaml

@@ -0,0 +1,6 @@
+pr: 109193
+summary: "[ES|QL] explicit cast a string literal to `date_period` and `time_duration`\
+  \ in arithmetic operations"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/111981.yaml

@@ -0,0 +1,6 @@
+pr: 111981
+summary: Allow fields with dots in sparse vector field mapper
+area: Mapping
+type: enhancement
+issues:
+ - 109118

docs/changelog/112294.yaml

@@ -0,0 +1,8 @@
+pr: 112294
+summary: "Use fallback synthetic source for `copy_to` and doc_values: false cases"
+area: Mapping
+type: enhancement
+issues:
+ - 110753
+ - 110038
+ - 109546

docs/changelog/112574.yaml

@@ -0,0 +1,5 @@
+pr: 112574
+summary: Add privileges required for CDR misconfiguration features to work on AWS SecurityHub integration
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/112649.yaml

@@ -0,0 +1,5 @@
+pr: 112649
+summary: Allowlist `tracestate` header on remote server port
+area: Security
+type: bug
+issues: []

docs/changelog/112713.yaml

@@ -0,0 +1,5 @@
+pr: 112713
+summary: Fix encoding of dynamic arrays in ignored source
+area: Logs
+type: bug
+issues: []

docs/changelog/112720.yaml

@@ -0,0 +1,5 @@
+pr: 112720
+summary: Fix NPE in `dense_vector` stats
+area: Vector Search
+type: bug
+issues: []

docs/reference/cluster/stats.asciidoc

@@ -1307,6 +1307,142 @@ Each repository type may also include other statistics about the repositories of
 
 ====
 
+`ccs`::
+(object) Contains information relating to <<modules-cross-cluster-search, {ccs}>> settings and activity in the cluster.
++
+.Properties of `ccs`
+[%collapsible%open]
+=====
+
+
+`_search`:::
+(object) Contains the telemetry information about the <<modules-cross-cluster-search, {ccs}>> usage in the cluster.
++
+.Properties of `_search`
+[%collapsible%open]
+======
+`total`:::
+(integer) The total number of {ccs} requests that have been executed by the cluster.
+
+`success`:::
+(integer) The total number of {ccs} requests that have been successfully executed by the cluster.
+
+`skipped`:::
+(integer) The total number of {ccs} requests (successful or failed) that had at least one remote cluster skipped.
+
+`took`:::
+(object) Contains statistics about the time taken to execute {ccs} requests.
++
+.Properties of `took`
+[%collapsible%open]
+=======
+`max`:::
+(integer) The maximum time taken to execute a {ccs} request, in milliseconds.
+
+`avg`:::
+(integer) The median time taken to execute a {ccs} request, in milliseconds.
+
+`p90`:::
+(integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+=======
+
+`took_mrt_true`::
+(object) Contains statistics about the time taken to execute {ccs} requests for which the
+<<ccs-minimize-roundtrips,`ccs_minimize_roundtrips`>> setting was set to `true`.
++
+.Properties of `took_mrt_true`
+[%collapsible%open]
+=======
+`max`:::
+(integer) The maximum time taken to execute a {ccs} request, in milliseconds.
+
+`avg`:::
+(integer) The median time taken to execute a {ccs} request, in milliseconds.
+
+`p90`:::
+(integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+=======
+
+`took_mrt_false`::
+(object) Contains statistics about the time taken to execute {ccs} requests for which the
+<<ccs-minimize-roundtrips,`ccs_minimize_roundtrips`>> setting was set to `false`.
++
+.Properties of `took_mrt_false`
+[%collapsible%open]
+=======
+`max`:::
+(integer) The maximum time taken to execute a {ccs} request, in milliseconds.
+
+`avg`:::
+(integer) The median time taken to execute a {ccs} request, in milliseconds.
+
+`p90`:::
+(integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+=======
+
+`remotes_per_search_max`::
+(integer) The maximum number of remote clusters that were queried in a single {ccs} request.
+
+`remotes_per_search_avg`::
+(float) The average number of remote clusters that were queried in a single {ccs} request.
+
+`failure_reasons`::
+(object) Contains statistics about the reasons for {ccs} request failures.
+The keys are the failure reason names and the values are the number of requests that failed for that reason.
+
+`features`::
+(object) Contains statistics about the features used in {ccs} requests. The keys are the names of the search feature,
+and the values are the number of requests that used that feature. Single request can use more than one feature
+(e.g. both `async` and `wildcard`). Known features are:
+
+* `async` - <<async-search, Async search>>
+
+* `mrt` - <<ccs-minimize-roundtrips,`ccs_minimize_roundtrips`>> setting was set to `true`.
+
+* `wildcard` - <<api-multi-index,Multi-target syntax>> for indices with wildcards was used in the search request.
+
+`clients`::
+(object) Contains statistics about the clients that executed {ccs} requests.
+The keys are the names of the clients, and the values are the number of requests that were executed by that client.
+Only known clients (such as `kibana` or `elasticsearch`) are counted.
+
+`clusters`::
+(object) Contains statistics about the clusters that were queried in {ccs} requests.
+The keys are cluster names, and the values are per-cluster telemetry data.
+This also includes the local cluster itself, which uses the name `(local)`.
++
+.Properties of per-cluster data:
+[%collapsible%open]
+=======
+`total`:::
+(integer) The total number of successful (not skipped) {ccs} requests that were executed against this cluster.
+This may include requests where partial results were returned, but not requests in which the cluster has been skipped entirely.
+
+`skipped`:::
+(integer) The total number of {ccs} requests for which this cluster was skipped.
+
+`took`:::
+(object) Contains statistics about the time taken to execute requests against this cluster.
++
+.Properties of `took`
+[%collapsible%open]
+========
+`max`:::
+(integer) The maximum time taken to execute a {ccs} request, in milliseconds.
+
+`avg`:::
+(integer) The median time taken to execute a {ccs} request, in milliseconds.
+
+`p90`:::
+(integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+========
+
+=======
+
+======
+
+=====
+
 [[cluster-stats-api-example]]
 ==== {api-examples-title}
 
@@ -1607,7 +1743,35 @@ The API returns the following response:
    },
    "repositories": {
      ...
-   }
+   },
+   "ccs": {
+     "_search": {
+        "total": 7,
+        "success": 7,
+        "skipped": 0,
+        "took": {
+            "max": 36,
+            "avg": 20,
+            "p90": 33
+        },
+        "took_mrt_true": {
+            "max": 33,
+            "avg": 15,
+            "p90": 33
+        },
+        "took_mrt_false": {
+            "max": 36,
+            "avg": 26,
+            "p90": 36
+        },
+        "remotes_per_search_max": 3,
+        "remotes_per_search_avg": 2.0,
+        "failure_reasons": { ... },
+        "features": { ... },
+        "clients": { ... },
+        "clusters": { ... }
+     }
+  }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"plugins": \[[^\]]*\]/"plugins": $body.$_path/]
@@ -1618,10 +1782,15 @@ The API returns the following response:
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
 // TESTRESPONSE[s/"snapshots": \{[^\}]*\}/"snapshots": $body.$_path/]
 // TESTRESPONSE[s/"repositories": \{[^\}]*\}/"repositories": $body.$_path/]
+// TESTRESPONSE[s/"clusters": \{[^\}]*\}/"clusters": $body.$_path/]
+// TESTRESPONSE[s/"features": \{[^\}]*\}/"features": $body.$_path/]
+// TESTRESPONSE[s/"clients": \{[^\}]*\}/"clients": $body.$_path/]
+// TESTRESPONSE[s/"failure_reasons": \{[^\}]*\}/"failure_reasons": $body.$_path/]
 // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
 // TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/]
 // TESTRESPONSE[s/"search": \{[^\}]*\}/"search": $body.$_path/]
-// TESTRESPONSE[s/: true|false/: $body.$_path/]
+// TESTRESPONSE[s/"remotes_per_search_avg": [.0-9]+/"remotes_per_search_avg": $body.$_path/]
+// TESTRESPONSE[s/: (true|false)/: $body.$_path/]
 // TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
 // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]
 // These replacements do a few things: