Merge branch '2.x' into backport/backport-16600-to-2.x

Signed-off-by: Michael Froh <[email protected]>

CHANGELOG.md

@@ -8,16 +8,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111))
 - Ensure support of the transport-nio by security plugin ([#16474](https://github.com/opensearch-project/OpenSearch/pull/16474))
 - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284))
+- Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files))
 - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
 - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583))
 - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600))
+- Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489))
 
 ### Dependencies
 - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548))
 - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521))
 - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611))
 - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610))
+- Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614))
 
 ### Changed
 

modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java

@@ -8,41 +8,187 @@
 
 package org.opensearch.search.pipeline.common;
 
-import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.admin.indices.refresh.RefreshRequest;
 import org.opensearch.action.admin.indices.refresh.RefreshResponse;
+import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.search.DeleteSearchPipelineRequest;
+import org.opensearch.action.search.GetSearchPipelineRequest;
+import org.opensearch.action.search.GetSearchPipelineResponse;
 import org.opensearch.action.search.PutSearchPipelineRequest;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.query.MatchAllQueryBuilder;
+import org.opensearch.ingest.PipelineConfiguration;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.After;
+import org.junit.Before;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 @OpenSearchIntegTestCase.SuiteScopeTestCase
 public class SearchPipelineCommonIT extends OpenSearchIntegTestCase {
 
+    private static final String TEST_INDEX = "myindex";
+    private static final String PIPELINE_NAME = "test_pipeline";
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return List.of(SearchPipelineCommonModulePlugin.class);
     }
 
+    @Before
+    public void setup() throws Exception {
+        createIndex(TEST_INDEX);
+
+        IndexRequest doc1 = new IndexRequest(TEST_INDEX).id("doc1").source(Map.of("field", "value"));
+        IndexRequest doc2 = new IndexRequest(TEST_INDEX).id("doc2").source(Map.of("field", "something else"));
+
+        IndexResponse ir = client().index(doc1).actionGet();
+        assertSame(RestStatus.CREATED, ir.status());
+        ir = client().index(doc2).actionGet();
+        assertSame(RestStatus.CREATED, ir.status());
+
+        RefreshResponse refRsp = client().admin().indices().refresh(new RefreshRequest(TEST_INDEX)).actionGet();
+        assertSame(RestStatus.OK, refRsp.getStatus());
+    }
+
+    @After
+    public void cleanup() throws Exception {
+        internalCluster().wipeIndices(TEST_INDEX);
+    }
+
     public void testFilterQuery() {
         // Create a pipeline with a filter_query processor.
-        String pipelineName = "foo";
+        createPipeline();
+
+        // Search without the pipeline. Should see both documents.
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        SearchResponse rsp = client().search(req).actionGet();
+        assertEquals(2, rsp.getHits().getTotalHits().value);
+
+        // Search with the pipeline. Should only see document with "field":"value".
+        req.pipeline(PIPELINE_NAME);
+        rsp = client().search(req).actionGet();
+        assertEquals(1, rsp.getHits().getTotalHits().value);
+
+        // Clean up.
+        deletePipeline();
+    }
+
+    public void testSearchWithTemporaryPipeline() throws Exception {
+
+        // Search without the pipeline. Should see both documents.
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        SearchResponse rsp = client().search(req).actionGet();
+        assertEquals(2, rsp.getHits().getTotalHits().value);
+
+        // Search with temporary pipeline
+        Map<String, Object> pipelineSourceMap = new HashMap<>();
+        Map<String, Object> requestProcessorConfig = new HashMap<>();
+
+        Map<String, Object> filterQuery = new HashMap<>();
+        filterQuery.put("query", Map.of("term", Map.of("field", "value")));
+        requestProcessorConfig.put("filter_query", filterQuery);
+        pipelineSourceMap.put("request_processors", List.of(requestProcessorConfig));
+
+        req = new SearchRequest(TEST_INDEX).source(
+            new SearchSourceBuilder().query(new MatchAllQueryBuilder()).searchPipelineSource(pipelineSourceMap)
+        );
+
+        SearchResponse rspWithTempPipeline = client().search(req).actionGet();
+        assertEquals(1, rspWithTempPipeline.getHits().getTotalHits().value);
+    }
+
+    public void testSearchWithDefaultPipeline() throws Exception {
+        // Create pipeline
+        createPipeline();
+
+        // Search without the pipeline. Should see both documents.
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        SearchResponse rsp = client().search(req).actionGet();
+        assertEquals(2, rsp.getHits().getTotalHits().value);
+
+        // Set pipeline as default for the index
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(TEST_INDEX);
+        updateSettingsRequest.settings(Settings.builder().put("index.search.default_pipeline", PIPELINE_NAME));
+        AcknowledgedResponse updateSettingsResponse = client().admin().indices().updateSettings(updateSettingsRequest).actionGet();
+        assertTrue(updateSettingsResponse.isAcknowledged());
+
+        // Search with the default pipeline. Should only see document with "field":"value".
+        rsp = client().search(req).actionGet();
+        assertEquals(1, rsp.getHits().getTotalHits().value);
+
+        // Clean up: Remove default pipeline setting
+        updateSettingsRequest = new UpdateSettingsRequest(TEST_INDEX);
+        updateSettingsRequest.settings(Settings.builder().putNull("index.search.default_pipeline"));
+        updateSettingsResponse = client().admin().indices().updateSettings(updateSettingsRequest).actionGet();
+        assertTrue(updateSettingsResponse.isAcknowledged());
+
+        // Clean up.
+        deletePipeline();
+    }
+
+    public void testUpdateSearchPipeline() throws Exception {
+        // Create initial pipeline
+        createPipeline();
+
+        // Verify initial pipeline
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        req.pipeline(PIPELINE_NAME);
+        SearchResponse initialRsp = client().search(req).actionGet();
+        assertEquals(1, initialRsp.getHits().getTotalHits().value);
+
+        BytesReference pipelineConfig = new BytesArray(
+            "{"
+                + "\"description\": \"Updated pipeline\","
+                + "\"request_processors\": ["
+                + "{"
+                + "\"filter_query\" : {"
+                + "\"query\": {"
+                + "\"term\" : {"
+                + "\"field\" : \"something else\""
+                + "}"
+                + "}"
+                + "}"
+                + "}"
+                + "]"
+                + "}"
+        );
+
+        PipelineConfiguration pipeline = new PipelineConfiguration(PIPELINE_NAME, pipelineConfig, MediaTypeRegistry.JSON);
+
+        // Update pipeline
+        PutSearchPipelineRequest updateRequest = new PutSearchPipelineRequest(pipeline.getId(), pipelineConfig, MediaTypeRegistry.JSON);
+        AcknowledgedResponse ackRsp = client().admin().cluster().putSearchPipeline(updateRequest).actionGet();
+        assertTrue(ackRsp.isAcknowledged());
+
+        // Verify pipeline description
+        GetSearchPipelineResponse getPipelineResponse = client().admin()
+            .cluster()
+            .getSearchPipeline(new GetSearchPipelineRequest(PIPELINE_NAME))
+            .actionGet();
+        assertEquals(PIPELINE_NAME, getPipelineResponse.pipelines().get(0).getId());
+        assertEquals(pipeline.getConfigAsMap(), getPipelineResponse.pipelines().get(0).getConfigAsMap());
+        // Clean up.
+        deletePipeline();
+    }
+
+    private void createPipeline() {
         PutSearchPipelineRequest putSearchPipelineRequest = new PutSearchPipelineRequest(
-            pipelineName,
+            PIPELINE_NAME,
             new BytesArray(
                 "{"
                     + "\"request_processors\": ["
@@ -62,35 +208,13 @@ public void testFilterQuery() {
         );
         AcknowledgedResponse ackRsp = client().admin().cluster().putSearchPipeline(putSearchPipelineRequest).actionGet();
         assertTrue(ackRsp.isAcknowledged());
+    }
 
-        // Index some documents.
-        String indexName = "myindex";
-        IndexRequest doc1 = new IndexRequest(indexName).id("doc1").source(Map.of("field", "value"));
-        IndexRequest doc2 = new IndexRequest(indexName).id("doc2").source(Map.of("field", "something else"));
-
-        IndexResponse ir = client().index(doc1).actionGet();
-        assertSame(RestStatus.CREATED, ir.status());
-        ir = client().index(doc2).actionGet();
-        assertSame(RestStatus.CREATED, ir.status());
-
-        // Refresh so the documents are visible to search.
-        RefreshResponse refRsp = client().admin().indices().refresh(new RefreshRequest(indexName)).actionGet();
-        assertSame(RestStatus.OK, refRsp.getStatus());
-
-        // Search without the pipeline. Should see both documents.
-        SearchRequest req = new SearchRequest(indexName).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
-        SearchResponse rsp = client().search(req).actionGet();
-        assertEquals(2, rsp.getHits().getTotalHits().value);
-
-        // Search with the pipeline. Should only see document with "field":"value".
-        req.pipeline(pipelineName);
-        rsp = client().search(req).actionGet();
-        assertEquals(1, rsp.getHits().getTotalHits().value);
-
-        // Clean up.
-        ackRsp = client().admin().cluster().deleteSearchPipeline(new DeleteSearchPipelineRequest(pipelineName)).actionGet();
-        assertTrue(ackRsp.isAcknowledged());
-        ackRsp = client().admin().indices().delete(new DeleteIndexRequest(indexName)).actionGet();
+    private void deletePipeline() {
+        AcknowledgedResponse ackRsp = client().admin()
+            .cluster()
+            .deleteSearchPipeline(new DeleteSearchPipelineRequest(PIPELINE_NAME))
+            .actionGet();
         assertTrue(ackRsp.isAcknowledged());
     }
 }

plugins/ingest-attachment/build.gradle

@@ -89,7 +89,7 @@ dependencies {
   api "org.apache.poi:poi:${versions.poi}"
   api "org.apache.poi:poi-ooxml-lite:${versions.poi}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'org.apache.xmlbeans:xmlbeans:5.2.1'
+  api 'org.apache.xmlbeans:xmlbeans:5.2.2'
   api 'org.apache.commons:commons-collections4:4.4'
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"

plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1

@@ -0,0 +1 @@
+586ffe10ae9864e19e85c24bd060790a70586f72

server/build.gradle

@@ -36,7 +36,7 @@ plugins {
   id('opensearch.publish')
   id('opensearch.internal-cluster-test')
   id('opensearch.optional-dependencies')
-  id('me.champeau.gradle.japicmp') version '0.4.3'
+  id('me.champeau.gradle.japicmp') version '0.4.5'
 }
 
 publishing {

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -787,6 +787,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 // Snapshot related Settings
                 BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING,
                 BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING,
+                BlobStoreRepository.SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD,
 
                 // Composite index settings
                 CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING,

server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java

@@ -60,7 +60,7 @@ public class SegmentReplicationPressureService implements Closeable {
 
     public static final Setting<Integer> MAX_INDEXING_CHECKPOINTS = Setting.intSetting(
         "segrep.pressure.checkpoint.limit",
-        4,
+        30,
         1,
         Setting.Property.Dynamic,
         Setting.Property.NodeScope