Merge branch 'main' into 1127/5-azure-pr

prwhelan · Oct 10, 2024 · 3d3907c · 3d3907c
2 parents 0419561 + 9ae0140
commit 3d3907c
Show file tree

Hide file tree

Showing 393 changed files with 10,313 additions and 4,188 deletions.
diff --git a/...onventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java b/...onventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java
@@ -44,7 +44,7 @@ public void apply(Project project) {
         gitInfo.disallowChanges();
         gitInfo.finalizeValueOnRead();
 
-        revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "master");
+        revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "main");
     }
 
     public Property<GitInfo> getGitInfo() {

diff --git a/...ventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java b/...ventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java
@@ -21,6 +21,7 @@
 public class LicensingPlugin implements Plugin<Project> {
     static final String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
     static final String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
+    static final String AGPL_ELASTIC_LICENSE_URL_POSTFIX = "/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt";
 
     private ProviderFactory providerFactory;
 
@@ -36,15 +37,18 @@ public void apply(Project project) {
              isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
         );
 
-        Provider<String> projectLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
+        Provider<String> elasticLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
                 licenseCommit + ELASTIC_LICENSE_URL_POSTFIX);
+        Provider<String> agplLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
+            licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX);
         // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
-        project.getExtensions().getExtraProperties().set("elasticLicenseUrl", projectLicenseURL);
+        project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL);
 
         MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class).convention(
                 providerFactory.provider(() -> Map.of(
                         "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license",
-                        "Elastic License 2.0", projectLicenseURL.get())
+                        "Elastic License 2.0", elasticLicenseURL.get(),
+                        "GNU Affero General Public License Version 3", agplLicenseURL.get())
                 )
         );
 

diff --git a/...ernal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/...ernal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
@@ -69,6 +69,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
       <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
       <distribution>repo</distribution>
     </license>
+    <license>
+      <name>GNU Affero General Public License Version 3</name>
+      <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+    </license>
     <license>
       <name>Server Side Public License, v 1</name>
       <url>https://www.mongodb.com/licensing/server-side-public-license</url>
@@ -144,6 +149,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                   <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
                   <distribution>repo</distribution>
                 </license>
+                <license>
+                  <name>GNU Affero General Public License Version 3</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
                 <license>
                   <name>Server Side Public License, v 1</name>
                   <url>https://www.mongodb.com/licensing/server-side-public-license</url>
@@ -228,6 +238,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                   <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
                   <distribution>repo</distribution>
                 </license>
+                <license>
+                  <name>GNU Affero General Public License Version 3</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
                 <license>
                   <name>Server Side Public License, v 1</name>
                   <url>https://www.mongodb.com/licensing/server-side-public-license</url>
@@ -321,6 +336,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                   <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
                   <distribution>repo</distribution>
                 </license>
+                <license>
+                  <name>GNU Affero General Public License Version 3</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
                 <license>
                   <name>Server Side Public License, v 1</name>
                   <url>https://www.mongodb.com/licensing/server-side-public-license</url>
@@ -394,6 +414,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                   <url>https://raw.githubusercontent.com/elastic/elasticsearch/v2.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
                   <distribution>repo</distribution>
                 </license>
+                <license>
+                  <name>GNU Affero General Public License Version 3</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v2.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
                 <license>
                   <name>Server Side Public License, v 1</name>
                   <url>https://www.mongodb.com/licensing/server-side-public-license</url>

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -31,7 +31,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984",
         "-wolfi",
         "apk"
     ),

diff --git a/distribution/docker/src/docker/Dockerfile.ess b/distribution/docker/src/docker/Dockerfile.ess
@@ -25,7 +25,7 @@ USER root
 
 COPY plugins/*.zip /opt/plugins/archive/
 
-RUN chown root.root /opt/plugins/archive/*
+RUN chown 1000:1000 /opt/plugins/archive/*
 RUN chmod 0444 /opt/plugins/archive/*
 
 FROM ${base_image}

diff --git a/docs/changelog/112905.yaml b/docs/changelog/112905.yaml
@@ -0,0 +1,5 @@
+pr: 112905
+summary: "[ES|QL] Named parameter for field names and field name patterns"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114109.yaml b/docs/changelog/114109.yaml
@@ -0,0 +1,5 @@
+pr: 114109
+summary: Update cluster stats for retrievers
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/114358.yaml b/docs/changelog/114358.yaml
@@ -0,0 +1,5 @@
+pr: 114358
+summary: "ESQL: Use less memory in listener"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114363.yaml b/docs/changelog/114363.yaml
@@ -0,0 +1,5 @@
+pr: 114363
+summary: Give the kibana system user permission to read security entities
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/114368.yaml b/docs/changelog/114368.yaml
@@ -0,0 +1,5 @@
+pr: 114368
+summary: "ESQL: Delay construction of warnings"
+area: EQL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114375.yaml b/docs/changelog/114375.yaml
@@ -0,0 +1,5 @@
+pr: 114375
+summary: Handle `InternalSendException` inline for non-forking handlers
+area: Distributed
+type: bug
+issues: []
diff --git a/docs/changelog/114386.yaml b/docs/changelog/114386.yaml
@@ -0,0 +1,5 @@
+pr: 114386
+summary: Improve handling of failure to create persistent task
+area: Task Management
+type: bug
+issues: []
diff --git a/docs/changelog/114389.yaml b/docs/changelog/114389.yaml
@@ -0,0 +1,5 @@
+pr: 114389
+summary: Filter empty task settings objects from the API response
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
@@ -762,6 +762,10 @@ Queries are counted once per search request, meaning that if the same query type
 (object) Search sections used in selected nodes.
 For each section, name and number of times it's been used is reported.
 
+`retrievers`::
+(object) Retriever types that were used in selected nodes.
+For each retriever, name and number of times it's been used is reported.
+
 =====
 
 `dense_vector`::

diff --git a/docs/reference/ingest/processors.asciidoc b/docs/reference/ingest/processors.asciidoc
@@ -185,6 +185,9 @@ Executes another pipeline.
 <<reroute-processor, `reroute` processor>>::
 Reroutes documents to another target index or data stream.
 
+<<terminate-processor, `terminate` processor>>::
+Terminates the current ingest pipeline, causing no further processors to be run.
+
 [discrete]
 [[ingest-process-category-array-json-handling]]
 === Array/JSON handling processors
@@ -258,6 +261,7 @@ include::processors/set.asciidoc[]
 include::processors/set-security-user.asciidoc[]
 include::processors/sort.asciidoc[]
 include::processors/split.asciidoc[]
+include::processors/terminate.asciidoc[]
 include::processors/trim.asciidoc[]
 include::processors/uppercase.asciidoc[]
 include::processors/url-decode.asciidoc[]

diff --git a/...treams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/...treams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
@@ -217,10 +216,7 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
             xContentRegistry(),
             EmptySystemIndices.INSTANCE,
             indexSettingProviders,
-            DataStreamGlobalRetentionSettings.create(
-                ClusterSettings.createBuiltInClusterSettings(),
-                DataStreamFactoryRetention.emptyFactoryRetention()
-            )
+            DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings())
         );
     }
 

diff --git a/...rc/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/...rc/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
@@ -47,8 +46,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
     private final IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance();
     private final SystemIndices systemIndices = new SystemIndices(List.of());
     private final DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = DataStreamGlobalRetentionSettings.create(
-        ClusterSettings.createBuiltInClusterSettings(),
-        DataStreamFactoryRetention.emptyFactoryRetention()
+        ClusterSettings.createBuiltInClusterSettings()
     );
 
     public void testGetDataStream() {
@@ -356,8 +354,7 @@ public void testPassingGlobalRetention() {
                     )
                     .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), globalRetention.maxRetention())
                     .build()
-            ),
-            DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
         response = TransportGetDataStreamsAction.innerOperation(
             state,

diff --git a/...rc/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/...rc/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling;
@@ -142,8 +141,7 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
     private DoExecuteDelegate clientDelegate;
     private ClusterService clusterService;
     private final DataStreamGlobalRetentionSettings globalRetentionSettings = DataStreamGlobalRetentionSettings.create(
-        ClusterSettings.createBuiltInClusterSettings(),
-        DataStreamFactoryRetention.emptyFactoryRetention()
+        ClusterSettings.createBuiltInClusterSettings()
     );
 
     @Before

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java
@@ -181,7 +181,11 @@ enum Database {
             Property.POSTAL_CODE
         ),
         Set.of(Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION)
-    ),;
+    ),
+    PrivacyDetection(
+        Set.of(Property.IP, Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE),
+        Set.of(Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE)
+    );
 
     private final Set<Property> properties;
     private final Set<Property> defaultProperties;
@@ -262,7 +266,13 @@ enum Property {
         TYPE,
         POSTAL_CODE,
         POSTAL_CONFIDENCE,
-        ACCURACY_RADIUS;
+        ACCURACY_RADIUS,
+        HOSTING,
+        TOR,
+        PROXY,
+        RELAY,
+        VPN,
+        SERVICE;
 
         /**
          * Parses a string representation of a property into an actual Property instance. Not all properties that exist are

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
@@ -58,6 +58,31 @@ static Long parseAsn(final String asn) {
         }
     }
 
+    /**
+     * Lax-ly parses a string that contains a boolean into a Boolean (or null, if such parsing isn't possible).
+     * @param bool a potentially empty (or null) string that is expected to contain a parsable boolean
+     * @return the parsed boolean
+     */
+    static Boolean parseBoolean(final String bool) {
+        if (bool == null) {
+            return null;
+        } else {
+            String trimmed = bool.toLowerCase(Locale.ROOT).trim();
+            if ("true".equals(trimmed)) {
+                return true;
+            } else if ("false".equals(trimmed)) {
+                // "false" can represent false -- this an expected future enhancement in how the database represents booleans
+                return false;
+            } else if (trimmed.isEmpty()) {
+                // empty string can represent false -- this is how the database currently represents 'false' values
+                return false;
+            } else {
+                logger.trace("Unable to parse non-compliant boolean string [{}]", bool);
+                return null;
+            }
+        }
+    }
+
     /**
      * Lax-ly parses a string that contains a double into a Double (or null, if such parsing isn't possible).
      * @param latlon a potentially empty (or null) string that is expected to contain a parsable double
@@ -132,6 +157,22 @@ public GeolocationResult(
         }
     }
 
+    public record PrivacyDetectionResult(Boolean hosting, Boolean proxy, Boolean relay, String service, Boolean tor, Boolean vpn) {
+        @SuppressWarnings("checkstyle:RedundantModifier")
+        @MaxMindDbConstructor
+        public PrivacyDetectionResult(
+            @MaxMindDbParameter(name = "hosting") String hosting,
+            // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
+            @MaxMindDbParameter(name = "proxy") String proxy,
+            @MaxMindDbParameter(name = "relay") String relay,
+            @MaxMindDbParameter(name = "service") String service, // n.b. this remains a string, the rest are parsed as booleans
+            @MaxMindDbParameter(name = "tor") String tor,
+            @MaxMindDbParameter(name = "vpn") String vpn
+        ) {
+            this(parseBoolean(hosting), parseBoolean(proxy), parseBoolean(relay), service, parseBoolean(tor), parseBoolean(vpn));
+        }
+    }
+
     static class Asn extends AbstractBase<AsnResult> {
         Asn(Set<Database.Property> properties) {
             super(properties, AsnResult.class);
@@ -286,6 +327,55 @@ protected Map<String, Object> transform(final Result<GeolocationResult> result)
         }
     }
 
+    static class PrivacyDetection extends AbstractBase<PrivacyDetectionResult> {
+        PrivacyDetection(Set<Database.Property> properties) {
+            super(properties, PrivacyDetectionResult.class);
+        }
+
+        @Override
+        protected Map<String, Object> transform(final Result<PrivacyDetectionResult> result) {
+            PrivacyDetectionResult response = result.result;
+
+            Map<String, Object> data = new HashMap<>();
+            for (Database.Property property : this.properties) {
+                switch (property) {
+                    case IP -> data.put("ip", result.ip);
+                    case HOSTING -> {
+                        if (response.hosting != null) {
+                            data.put("hosting", response.hosting);
+                        }
+                    }
+                    case TOR -> {
+                        if (response.tor != null) {
+                            data.put("tor", response.tor);
+                        }
+                    }
+                    case PROXY -> {
+                        if (response.proxy != null) {
+                            data.put("proxy", response.proxy);
+                        }
+                    }
+                    case RELAY -> {
+                        if (response.relay != null) {
+                            data.put("relay", response.relay);
+                        }
+                    }
+                    case VPN -> {
+                        if (response.vpn != null) {
+                            data.put("vpn", response.vpn);
+                        }
+                    }
+                    case SERVICE -> {
+                        if (Strings.hasText(response.service)) {
+                            data.put("service", response.service);
+                        }
+                    }
+                }
+            }
+            return data;
+        }
+    }
+
     /**
      * Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the
      * getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for