From 9aad3a3b7f753e2ab69c1d3f834e4e21af372380 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 13:21:52 -0400 Subject: [PATCH 001/186] Bump org.apache.hadoop:hadoop-minicluster from 3.3.5 to 3.3.6 in /test/fixtures/hdfs-fixture (#8257) * Bump org.apache.hadoop:hadoop-minicluster in /test/fixtures/hdfs-fixture Bumps org.apache.hadoop:hadoop-minicluster from 3.3.5 to 3.3.6. --- updated-dependencies: - dependency-name: org.apache.hadoop:hadoop-minicluster dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 3 ++- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c794135cb23..8f830334f7897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140)) - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181)) - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) +- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) @@ -139,4 +140,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 70b84a405c9c6..de6f69a4fd4ce 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -37,7 +37,7 @@ versions << [ ] dependencies { - api("org.apache.hadoop:hadoop-minicluster:3.3.5") { + api("org.apache.hadoop:hadoop-minicluster:3.3.6") { exclude module: 'websocket-client' exclude module: 'jettison' exclude module: 'netty' From 3b2a93f9fc89cdf5b19c8af0381e19ff030d4374 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:10:51 -0400 Subject: [PATCH 002/186] Bump com.networknt:json-schema-validator from 1.0.84 to 1.0.85 in /buildSrc (#8255) * Bump com.networknt:json-schema-validator in /buildSrc Bumps [com.networknt:json-schema-validator](https://github.com/networknt/json-schema-validator) from 1.0.84 to 1.0.85. - [Release notes](https://github.com/networknt/json-schema-validator/releases) - [Changelog](https://github.com/networknt/json-schema-validator/blob/master/CHANGELOG.md) - [Commits](https://github.com/networknt/json-schema-validator/compare/1.0.84...1.0.85) --- updated-dependencies: - dependency-name: com.networknt:json-schema-validator dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f830334f7897..cb2a75da2454d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,7 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901)) - Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) - Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) -- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.83 ([7968](https://github.com/opensearch-project/OpenSearch/pull/7968)) +- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.85 ([7968], #8255) - Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068)) - Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre (#8009) - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 (#8014, #8013, #8010) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index abbdc6f6a570e..eca536e6e90cf 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -118,7 +118,7 @@ dependencies { api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.9.2' - api 'com.networknt:json-schema-validator:1.0.84' + api 'com.networknt:json-schema-validator:1.0.85' api 'org.jruby.jcodings:jcodings:1.0.58' api 'org.jruby.joni:joni:2.1.48' api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" From c29e4aa774199b007acffd17bb016f4bfc681874 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:52:52 -0400 Subject: [PATCH 003/186] Bump io.projectreactor.netty:reactor-netty-http from 1.1.7 to 1.1.8 in /plugins/repository-azure (#8256) * Bump io.projectreactor.netty:reactor-netty-http Bumps [io.projectreactor.netty:reactor-netty-http](https://github.com/reactor/reactor-netty) from 1.1.7 to 1.1.8. - [Release notes](https://github.com/reactor/reactor-netty/releases) - [Commits](https://github.com/reactor/reactor-netty/compare/v1.1.7...v1.1.8) --- updated-dependencies: - dependency-name: io.projectreactor.netty:reactor-netty-http dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../repository-azure/licenses/reactor-netty-http-1.1.7.jar.sha1 | 1 - .../repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.7.jar.sha1 create mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index cb2a75da2454d..12d57af9018f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181)) - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257) +- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 48a49af165542..e67ea7ab0a11e 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -60,7 +60,7 @@ dependencies { api 'io.projectreactor:reactor-core:3.5.6' api 'io.projectreactor.netty:reactor-netty:1.1.7' api 'io.projectreactor.netty:reactor-netty-core:1.1.7' - api 'io.projectreactor.netty:reactor-netty-http:1.1.7' + api 'io.projectreactor.netty:reactor-netty-http:1.1.8' api "org.slf4j:slf4j-api:${versions.slf4j}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.7.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.7.jar.sha1 deleted file mode 100644 index 33bf2aabcfc9b..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-http-1.1.7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -39d7c0a13afa471b426a30bcf82664496ad34723 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1 new file mode 100644 index 0000000000000..5092608c90eba --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1 @@ -0,0 +1 @@ +696ea25658295e49906c6aad13fa70acbdeb2359 \ No newline at end of file From e802b538beb4f0071c6d28ceae655d2b26c70236 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 13:13:43 -0700 Subject: [PATCH 004/186] Bump org.jruby.joni:joni from 2.1.48 to 2.2.1 (#8254) Bumps [org.jruby.joni:joni](https://github.com/jruby/joni) from 2.1.48 to 2.2.1. - [Commits](https://github.com/jruby/joni/compare/joni-2.1.48...joni-2.2.1) --- updated-dependencies: - dependency-name: org.jruby.joni:joni dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12d57af9018f5..f4db81ecf16e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,7 +101,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 (#8014, #8013, #8010) - Bump `com.diffplug.spotless` from 6.18.0 to 6.19.0 (#8007) - Bump `'com.azure:azure-storage-blob` to 12.22.2 from 12.21.1 ([#8043](https://github.com/opensearch-project/OpenSearch/pull/8043)) -- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015) +- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015, #8254) - Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre ([#8011](https://github.com/opensearch-project/OpenSearch/pull/8011), [#8012](https://github.com/opensearch-project/OpenSearch/pull/8012), [#8107](https://github.com/opensearch-project/OpenSearch/pull/8107)) - Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016)) - Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index eca536e6e90cf..35f3fb87560e7 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -120,7 +120,7 @@ dependencies { api 'org.apache.maven:maven-model:3.9.2' api 'com.networknt:json-schema-validator:1.0.85' api 'org.jruby.jcodings:jcodings:1.0.58' - api 'org.jruby.joni:joni:2.1.48' + api 'org.jruby.joni:joni:2.2.1' api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" api "org.ajoberstar.grgit:grgit-core:5.2.0" From 57798de012135d9c223957933c3ce207297ba13f Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Tue, 27 Jun 2023 04:54:15 +0530 Subject: [PATCH 005/186] Add integ tests for remote store stats api (#8135) Signed-off-by: bansvaru --- .../RemoteStoreBaseIntegTestCase.java | 22 +++- .../remotestore/RemoteStoreStatsIT.java | 115 ++++++++++++++---- 2 files changed, 106 insertions(+), 31 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 0ffa5ab23e0b6..d226d0d757638 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -62,22 +62,36 @@ private Settings defaultIndexSettings() { .build(); } - protected Settings remoteStoreIndexSettings(int numberOfReplicas) { + protected Settings remoteStoreIndexSettings(int numberOfReplicas, int numberOfShards) { return Settings.builder() .put(defaultIndexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) .build(); } - protected Settings remoteTranslogIndexSettings(int numberOfReplicas) { + protected Settings remoteStoreIndexSettings(int numberOfReplicas) { + return remoteStoreIndexSettings(numberOfReplicas, 1); + } + + protected Settings remoteTranslogIndexSettings(int numberOfReplicas, int numberOfShards) { return Settings.builder() - .put(remoteStoreIndexSettings(numberOfReplicas)) + .put(remoteStoreIndexSettings(numberOfReplicas, numberOfShards)) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .build(); } + protected Settings remoteTranslogIndexSettings(int numberOfReplicas) { + return remoteTranslogIndexSettings(numberOfReplicas, 1); + } + + protected void putRepository(Path path) { + assertAcked( + clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", path)) + ); + } + @Before public void setup() { internalCluster().startClusterManagerOnlyNode(); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 3c5853f9a64e9..0ea87d106c14e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -9,18 +9,21 @@ package org.opensearch.remotestore; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; +import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsRequestBuilder; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.UUIDs; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.test.OpenSearchIntegTestCase; import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 3) public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -29,7 +32,6 @@ public void testStatsResponseFromAllNodes() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. - internalCluster().startDataOnlyNodes(3); if (randomBoolean()) { createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); } else { @@ -38,18 +40,7 @@ public void testStatsResponseFromAllNodes() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - // Indexing documents along with refreshes and flushes. - for (int i = 0; i < randomIntBetween(5, 10); i++) { - if (randomBoolean()) { - flush(INDEX_NAME); - } else { - refresh(INDEX_NAME); - } - int numberOfOperations = randomIntBetween(20, 50); - for (int j = 0; j < numberOfOperations; j++) { - indexSingleDoc(); - } - } + indexDocs(); // Step 2 - We find all the nodes that are present in the cluster. We make the remote store stats api call from // each of the node in the cluster and check that the response is coming as expected. @@ -66,23 +57,93 @@ public void testStatsResponseFromAllNodes() { .collect(Collectors.toList()); assertEquals(1, matches.size()); RemoteRefreshSegmentTracker.Stats stats = matches.get(0).getStats(); - assertEquals(0, stats.refreshTimeLagMs); - assertEquals(stats.localRefreshNumber, stats.remoteRefreshNumber); - assertTrue(stats.uploadBytesStarted > 0); - assertEquals(0, stats.uploadBytesFailed); - assertTrue(stats.uploadBytesSucceeded > 0); - assertTrue(stats.totalUploadsStarted > 0); - assertEquals(0, stats.totalUploadsFailed); - assertTrue(stats.totalUploadsSucceeded > 0); - assertEquals(0, stats.rejectionCount); - assertEquals(0, stats.consecutiveFailuresCount); - assertEquals(0, stats.bytesLag); - assertTrue(stats.uploadBytesMovingAverage > 0); - assertTrue(stats.uploadBytesPerSecMovingAverage > 0); - assertTrue(stats.uploadTimeMovingAverage > 0); + assertResponseStats(stats); + } + } + + public void testStatsResponseAllShards() { + + // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes + // during this time frame. This ensures that the segment upload has started. + createIndex(INDEX_NAME, remoteTranslogIndexSettings(0, 3)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + indexDocs(); + + // Step 2 - We find all the nodes that are present in the cluster. We make the remote store stats api call from + // each of the node in the cluster and check that the response is coming as expected. + ClusterState state = getClusterState(); + String node = state.nodes().getDataNodes().values().stream().map(DiscoveryNode::getName).findFirst().get(); + RemoteStoreStatsRequestBuilder remoteStoreStatsRequestBuilder = client(node).admin() + .cluster() + .prepareRemoteStoreStats(INDEX_NAME, null); + RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); + assertTrue(response.getSuccessfulShards() == 3); + assertTrue(response.getShards() != null && response.getShards().length == 3); + RemoteRefreshSegmentTracker.Stats stats = response.getShards()[0].getStats(); + assertResponseStats(stats); + } + + public void testStatsResponseFromLocalNode() { + + // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes + // during this time frame. This ensures that the segment upload has started. + createIndex(INDEX_NAME, remoteTranslogIndexSettings(0, 3)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + indexDocs(); + + // Step 2 - We find a data node in the cluster. We make the remote store stats api call from + // each of the data node in the cluster and check that only local shards are returned. + ClusterState state = getClusterState(); + List nodes = state.nodes().getDataNodes().values().stream().map(DiscoveryNode::getName).collect(Collectors.toList()); + for (String node : nodes) { + RemoteStoreStatsRequestBuilder remoteStoreStatsRequestBuilder = client(node).admin() + .cluster() + .prepareRemoteStoreStats(INDEX_NAME, null); + remoteStoreStatsRequestBuilder.setLocal(true); + RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); + assertTrue(response.getSuccessfulShards() == 1); + assertTrue(response.getShards() != null && response.getShards().length == 1); + RemoteRefreshSegmentTracker.Stats stats = response.getShards()[0].getStats(); + assertResponseStats(stats); + } + } + + private void indexDocs() { + // Indexing documents along with refreshes and flushes. + for (int i = 0; i < randomIntBetween(5, 10); i++) { + if (randomBoolean()) { + flush(INDEX_NAME); + } else { + refresh(INDEX_NAME); + } + int numberOfOperations = randomIntBetween(20, 50); + for (int j = 0; j < numberOfOperations; j++) { + indexSingleDoc(); + } } } + private void assertResponseStats(RemoteRefreshSegmentTracker.Stats stats) { + assertEquals(0, stats.refreshTimeLagMs); + assertEquals(stats.localRefreshNumber, stats.remoteRefreshNumber); + assertTrue(stats.uploadBytesStarted > 0); + assertEquals(0, stats.uploadBytesFailed); + assertTrue(stats.uploadBytesSucceeded > 0); + assertTrue(stats.totalUploadsStarted > 0); + assertEquals(0, stats.totalUploadsFailed); + assertTrue(stats.totalUploadsSucceeded > 0); + assertEquals(0, stats.rejectionCount); + assertEquals(0, stats.consecutiveFailuresCount); + assertEquals(0, stats.bytesLag); + assertTrue(stats.uploadBytesMovingAverage > 0); + assertTrue(stats.uploadBytesPerSecMovingAverage > 0); + assertTrue(stats.uploadTimeMovingAverage > 0); + } + private IndexResponse indexSingleDoc() { return client().prepareIndex(INDEX_NAME) .setId(UUIDs.randomBase64UUID()) From 45a934d38ad1c7350010a4537322d03bbea9826f Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Mon, 26 Jun 2023 19:07:43 -0500 Subject: [PATCH 006/186] [Upgrade] Lucene 9.7.0 release (#8272) * [Upgrade] Lucene 9.7.0 release Upgrades to the official 9.7.0 release of lucene Signed-off-by: Nicholas Walter Knize * update changelog Signed-off-by: Nicholas Walter Knize --------- Signed-off-by: Nicholas Walter Knize --- CHANGELOG.md | 3 ++- buildSrc/version.properties | 2 +- libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 | 1 - libs/core/licenses/lucene-core-9.7.0.jar.sha1 | 1 + .../lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 | 1 + .../lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 | 1 + .../lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 | 1 + .../lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../licenses/lucene-analysis-stempel-9.7.0.jar.sha1 | 1 + .../lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1 | 1 - .../licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 | 1 + .../lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-analysis-common-9.7.0.jar.sha1 | 1 + .../lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 | 1 + server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-core-9.7.0.jar.sha1 | 1 + .../licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-grouping-9.7.0.jar.sha1 | 1 + .../lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-highlighter-9.7.0.jar.sha1 | 1 + server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-join-9.7.0.jar.sha1 | 1 + server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-memory-9.7.0.jar.sha1 | 1 + server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-misc-9.7.0.jar.sha1 | 1 + server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-queries-9.7.0.jar.sha1 | 1 + .../lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-queryparser-9.7.0.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.7.0.jar.sha1 | 1 + .../lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 | 1 + .../licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-spatial3d-9.7.0.jar.sha1 | 1 + server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1 | 1 - server/licenses/lucene-suggest-9.7.0.jar.sha1 | 1 + .../src/main/java/org/opensearch/index/search/MatchQuery.java | 1 - .../main/java/org/opensearch/index/search/MultiMatchQuery.java | 1 - 50 files changed, 26 insertions(+), 27 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.7.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-core-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-join-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.7.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.7.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index f4db81ecf16e5..f5bc63794e46d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257) - Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256) +- [Upgrade] Lucene 9.7.0 release (#8272) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) @@ -141,4 +142,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x diff --git a/buildSrc/version.properties b/buildSrc/version.properties index d3dbae38c2615..377421703b892 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.7.0-snapshot-204acc3 +lucene = 9.7.0 bundled_jdk_vendor = adoptium bundled_jdk = 20.0.1+9 diff --git a/libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 2afe66c03cf22..0000000000000 --- a/libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3aa698cf90f074cbf24acfd7feaaad84c5a6f829 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..2b0f77275c0ab --- /dev/null +++ b/libs/core/licenses/lucene-core-9.7.0.jar.sha1 @@ -0,0 +1 @@ +ad391210ffd806931334be9670a35af00c56f959 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index f70c327ab5f6b..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c31671978777d97c026b13decfbef1d7eeed4410 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..ecf696b4b3b83 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 @@ -0,0 +1 @@ +297e1cfade4ef71466cc9d4f361d81807c8dc4c8 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index b5c59c8f5ed8c..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e31f7c161ad2b7652b1ac9100b4fa67d52be86eb \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..0ed030926ab93 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 @@ -0,0 +1 @@ +94293b169fb8572f440a5a4a523320ecf9778ffe \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index e3f1d7afdc72a..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed13aaa5843c76ba8bb9fcd1cac6de8855ea7f1c \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..ddd67276606a5 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 @@ -0,0 +1 @@ +2df800a38b64867b8dcd61fc2cd986114e4a80cb \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 12210d41b4e6f..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f8673a2b86d3868e9ef2745d1d3fa9ad5a63e84f \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..0cd68af98e724 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 @@ -0,0 +1 @@ +a01e8153f34d72e8c8c0180c1dea5b10f677dd3a \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 307d90ca834b6..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f7a585d5f62033f19a5ec79b66a74cc24c9e9d10 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..c7b4d2dc6da75 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 @@ -0,0 +1 @@ +b7d47d54683b0b1e09b271c32d1b7d3eb1990f49 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index b4e595a07ffda..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -40ba622c3aa91a0a8f2747617fef8f59bf212345 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..8df7245044171 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 @@ -0,0 +1 @@ +5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 9b6dcc21465fa..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e08432a3db0bde29f425812896bd695c5477b05f \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..974e4202f5ffb --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 @@ -0,0 +1 @@ +d23b1f05b471e05d0d6068b3ece7c8c65672eae7 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index b70b4e8d6c35b..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a44008e83286060f0265a32f9a9b586e86ec03b1 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..dce408a7d40ef --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 @@ -0,0 +1 @@ +dfb4313f3c68d337310522840d7144c1605d084a \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index a770070d97c2d..0000000000000 --- a/server/licenses/lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5aa5989b931c68eee90b22bca3f1f280a7d5c1ee \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..45d8f459573b1 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 @@ -0,0 +1 @@ +27ba6caaa4587a982cd451f7217b5a982bcfc44a \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index c35bc03c19097..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2326fb4762cd14eff6bea770d5c85a848e7a2482 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..3981ea4fa226e --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 @@ -0,0 +1 @@ +6389463bfbfcf902c8d31d12e9513a6818ac9d5e \ No newline at end of file diff --git a/server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 2afe66c03cf22..0000000000000 --- a/server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3aa698cf90f074cbf24acfd7feaaad84c5a6f829 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..2b0f77275c0ab --- /dev/null +++ b/server/licenses/lucene-core-9.7.0.jar.sha1 @@ -0,0 +1 @@ +ad391210ffd806931334be9670a35af00c56f959 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index c7aaff3a0b184..0000000000000 --- a/server/licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3251c58b21c5c205c63bbe618c0d3c1393908e1 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..90acbf6dcee8d --- /dev/null +++ b/server/licenses/lucene-grouping-9.7.0.jar.sha1 @@ -0,0 +1 @@ +8e6f0c229f4861be641047c33b05067176e4279c \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 6f9bfe7604f7f..0000000000000 --- a/server/licenses/lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2313a0755a5774eb5dfcdc116dc15bd9367c8cc1 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..bfcca0bc6cb5b --- /dev/null +++ b/server/licenses/lucene-highlighter-9.7.0.jar.sha1 @@ -0,0 +1 @@ +facb7c7ee0f75ed457a2d98f10d6430e25a53691 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 00b5c33be3568..0000000000000 --- a/server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -25fddcaf47d5614f48dc264f79151eb87990abd9 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..0dab3a7ddc41a --- /dev/null +++ b/server/licenses/lucene-join-9.7.0.jar.sha1 @@ -0,0 +1 @@ +d041bdc0947a14223cf68357407ee18b21027587 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index cf7eced4686aa..0000000000000 --- a/server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7c2aad4c1baa818a94bd787615e70824e6ae3a9d \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..357a9c4b2ea26 --- /dev/null +++ b/server/licenses/lucene-memory-9.7.0.jar.sha1 @@ -0,0 +1 @@ +0fade51ee353e15ddbbc45262aafe6f99ed020f1 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index b5ac7eb1f988b..0000000000000 --- a/server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -56211acb2a31b1967568eba9953db502a53f9010 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..da5e1921626b2 --- /dev/null +++ b/server/licenses/lucene-misc-9.7.0.jar.sha1 @@ -0,0 +1 @@ +7fcf451e2376526c3a027958812866cc5b0ff13f \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 2aa628ebc9573..0000000000000 --- a/server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c45b1847150e2e2e6302177b41faf95df2ca4fbf \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..fa82e95a7e19f --- /dev/null +++ b/server/licenses/lucene-queries-9.7.0.jar.sha1 @@ -0,0 +1 @@ +126989d4622419aa06fcbf3a342e859cab8c8799 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 5e7b5d9da9c0d..0000000000000 --- a/server/licenses/lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d40fe6400f21564ccb50b6716ec42d8dcd526b9d \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..438db0aea66e1 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.7.0.jar.sha1 @@ -0,0 +1 @@ +6e77bde908ff698354e4a2149e6dd4658b56d7b0 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index e45e9b929836b..0000000000000 --- a/server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9347773d0b269768c58dd0b438bdc4b450f9185 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..38b0b1cccbc29 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.7.0.jar.sha1 @@ -0,0 +1 @@ +9f3e8e1947f2f1c5784132444af51a060ff0b4bf \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index a697cb9fa2dff..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7055502c9a77b9a93d1b96c99397fd765dd7891f \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..48679df469fd1 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 @@ -0,0 +1 @@ +01b0bc7a407d8c35a70a1adf7966bb3e7caae928 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 3b96e448add5b..0000000000000 --- a/server/licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a2c5a7be5887dacf89c251936c3f3388ca20d28e \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..55d4d217fa6b9 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 @@ -0,0 +1 @@ +7c6b1b6e0a70c9cd177371e648648c2f896742a2 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1 deleted file mode 100644 index 0460861900897..0000000000000 --- a/server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1a076a8db84fa85634cc4894ee4fc78b5e69525c \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1 new file mode 100644 index 0000000000000..d4d7e6cd6bed9 --- /dev/null +++ b/server/licenses/lucene-suggest-9.7.0.jar.sha1 @@ -0,0 +1 @@ +5c37fd9a5d71dc87fe1cd4c18ff295ec8cfac170 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/search/MatchQuery.java b/server/src/main/java/org/opensearch/index/search/MatchQuery.java index 2c8b091fef8ef..91a4b456bfa2a 100644 --- a/server/src/main/java/org/opensearch/index/search/MatchQuery.java +++ b/server/src/main/java/org/opensearch/index/search/MatchQuery.java @@ -56,7 +56,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.QueryBuilder; -import org.apache.lucene.util.TermAndBoost; import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings; import org.opensearch.OpenSearchException; import org.opensearch.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java b/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java index 554d9714409b7..241f05af2c512 100644 --- a/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java +++ b/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java @@ -42,7 +42,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermAndBoost; import org.opensearch.common.lucene.search.Queries; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.AbstractQueryBuilder; From 0c7ba945806893ab07c705324de890f21bd623cc Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 27 Jun 2023 08:04:23 +0530 Subject: [PATCH 007/186] Fix SegmentReplication flaky integ tests (#8134) Signed-off-by: Sachin Kale --- .../replication/SegmentReplicationIT.java | 91 ++++++++++--------- .../SegmentReplicationRemoteStoreIT.java | 4 +- .../opensearch/index/shard/IndexShard.java | 30 ++++-- .../RemoteStoreRefreshListenerTests.java | 1 - 4 files changed, 70 insertions(+), 56 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 0a593e6149ddd..ce5e0989b622f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; +import org.junit.Before; import org.opensearch.action.ActionFuture; import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; @@ -95,11 +96,16 @@ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class SegmentReplicationIT extends SegmentReplicationBaseIT { + @Before + private void setup() { + internalCluster().startClusterManagerOnlyNode(); + } + public void testPrimaryStopped_ReplicaPromoted() throws Exception { - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); @@ -125,7 +131,7 @@ public void testPrimaryStopped_ReplicaPromoted() throws Exception { assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); // start another node, index another doc and replicate. - String nodeC = internalCluster().startNode(); + String nodeC = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); client().prepareIndex(INDEX_NAME).setId("4").setSource("baz", "baz").get(); refresh(INDEX_NAME); @@ -134,10 +140,10 @@ public void testPrimaryStopped_ReplicaPromoted() throws Exception { } public void testRestartPrimary() throws Exception { - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); assertEquals(getNodeContainingPrimaryShard().getName(), primary); @@ -160,10 +166,10 @@ public void testRestartPrimary() throws Exception { public void testCancelPrimaryAllocation() throws Exception { // this test cancels allocation on the primary - promoting the new replica and recreating the former primary as a replica. - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); final int initialDocCount = 1; @@ -190,8 +196,8 @@ public void testCancelPrimaryAllocation() throws Exception { } public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { - final String nodeA = internalCluster().startNode(); - final String nodeB = internalCluster().startNode(); + final String nodeA = internalCluster().startDataOnlyNode(); + final String nodeB = internalCluster().startDataOnlyNode(); final Settings settings = Settings.builder() .put(indexSettings()) .put( @@ -233,8 +239,8 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { } public void testIndexReopenClose() throws Exception { - final String primary = internalCluster().startNode(); - final String replica = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); + final String replica = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -274,8 +280,8 @@ public void testMultipleShards() throws Exception { .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); - final String nodeA = internalCluster().startNode(); - final String nodeB = internalCluster().startNode(); + final String nodeA = internalCluster().startDataOnlyNode(); + final String nodeB = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME, indexSettings); ensureGreen(INDEX_NAME); @@ -310,8 +316,8 @@ public void testMultipleShards() throws Exception { } public void testReplicationAfterForceMerge() throws Exception { - final String nodeA = internalCluster().startNode(); - final String nodeB = internalCluster().startNode(); + final String nodeA = internalCluster().startDataOnlyNode(); + final String nodeB = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -351,14 +357,13 @@ public void testReplicationAfterForceMerge() throws Exception { * This test verifies that segment replication does not fail for closed indices */ public void testClosedIndices() { - internalCluster().startClusterManagerOnlyNode(); List nodes = new ArrayList<>(); // start 1st node so that it contains the primary - nodes.add(internalCluster().startNode()); + nodes.add(internalCluster().startDataOnlyNode()); createIndex(INDEX_NAME, super.indexSettings()); ensureYellowAndNoInitializingShards(INDEX_NAME); // start 2nd node so that it contains the replica - nodes.add(internalCluster().startNode()); + nodes.add(internalCluster().startDataOnlyNode()); ensureGreen(INDEX_NAME); logger.info("--> Close index"); @@ -373,8 +378,7 @@ public void testClosedIndices() { * @throws Exception when issue is encountered */ public void testNodeDropWithOngoingReplication() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); createIndex( INDEX_NAME, Settings.builder() @@ -385,7 +389,7 @@ public void testNodeDropWithOngoingReplication() throws Exception { .build() ); ensureYellow(INDEX_NAME); - final String replicaNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); // Get replica allocation id @@ -447,11 +451,11 @@ public void testNodeDropWithOngoingReplication() throws Exception { } public void testCancellation() throws Exception { - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()); ensureYellow(INDEX_NAME); - final String replicaNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startDataOnlyNode(); final SegmentReplicationSourceService segmentReplicationSourceService = internalCluster().getInstance( SegmentReplicationSourceService.class, @@ -506,7 +510,7 @@ public void testCancellation() throws Exception { } public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()); ensureGreen(INDEX_NAME); @@ -529,7 +533,7 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { .prepareUpdateSettings(INDEX_NAME) .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) ); - final String replicaNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2); @@ -544,8 +548,8 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { } public void testDeleteOperations() throws Exception { - final String nodeA = internalCluster().startNode(); - final String nodeB = internalCluster().startNode(); + final String nodeA = internalCluster().startDataOnlyNode(); + final String nodeB = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -591,9 +595,9 @@ public void testDeleteOperations() throws Exception { */ public void testReplicationPostDeleteAndForceMerge() throws Exception { assumeFalse("Skipping the test with Remote store as its flaky.", segmentReplicationWithRemoteEnabled()); - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); final int initialDocCount = scaledRandomIntBetween(10, 200); for (int i = 0; i < initialDocCount; i++) { @@ -648,7 +652,6 @@ public void testReplicationPostDeleteAndForceMerge() throws Exception { } public void testUpdateOperations() throws Exception { - internalCluster().startClusterManagerOnlyNode(); final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellow(INDEX_NAME); @@ -702,7 +705,6 @@ public void testDropPrimaryDuringReplication() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replica_count) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); - final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); final String primaryNode = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME, settings); final List dataNodes = internalCluster().startDataOnlyNodes(6); @@ -742,11 +744,10 @@ public void testDropPrimaryDuringReplication() throws Exception { } public void testReplicaHasDiffFilesThanPrimary() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()); ensureYellow(INDEX_NAME); - final String replicaNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); @@ -796,9 +797,9 @@ public void testReplicaHasDiffFilesThanPrimary() throws Exception { } public void testPressureServiceStats() throws Exception { - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); - final String replicaNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); int initialDocCount = scaledRandomIntBetween(100, 200); @@ -848,7 +849,7 @@ public void testPressureServiceStats() throws Exception { assertEquals(0, replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats().size()); // start another replica. - String replicaNode_2 = internalCluster().startNode(); + String replicaNode_2 = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); String docId = String.valueOf(initialDocCount + 1); client().prepareIndex(INDEX_NAME).setId(docId).setSource("foo", "bar").get(); @@ -887,10 +888,10 @@ public void testPressureServiceStats() throws Exception { public void testScrollCreatedOnReplica() throws Exception { assumeFalse("Skipping the test with Remote store as its flaky.", segmentReplicationWithRemoteEnabled()); // create the cluster with one primary node containing primary shard and replica node containing replica shard - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); // index 100 docs @@ -981,7 +982,7 @@ public void testScrollWithOngoingSegmentReplication() throws Exception { ); // create the cluster with one primary node containing primary shard and replica node containing replica shard - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); prepareCreate( INDEX_NAME, Settings.builder() @@ -989,7 +990,7 @@ public void testScrollWithOngoingSegmentReplication() throws Exception { .put("index.refresh_interval", -1) ).get(); ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); final int initialDocCount = 10; @@ -1104,10 +1105,10 @@ public void testScrollWithOngoingSegmentReplication() throws Exception { } public void testPitCreatedOnReplica() throws Exception { - final String primary = internalCluster().startNode(); + final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replica = internalCluster().startNode(); + final String replica = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); client().prepareIndex(INDEX_NAME) .setId("1") @@ -1234,13 +1235,13 @@ public void testPitCreatedOnReplica() throws Exception { */ public void testPrimaryReceivesDocsDuringReplicaRecovery() throws Exception { final List nodes = new ArrayList<>(); - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); nodes.add(primaryNode); final Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); createIndex(INDEX_NAME, settings); ensureGreen(INDEX_NAME); // start a replica node, initially will be empty with no shard assignment. - final String replicaNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startDataOnlyNode(); nodes.add(replicaNode); // index a doc. diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java index ab0c0cc3aec77..7e79812fcfeea 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java @@ -8,7 +8,6 @@ package org.opensearch.remotestore; -import org.apache.lucene.tests.util.LuceneTestCase; import org.junit.After; import org.junit.Before; import org.opensearch.cluster.metadata.IndexMetadata; @@ -26,7 +25,6 @@ * This makes sure that the constructs/flows that are being tested with Segment Replication, holds true after enabling * remote store. */ -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7643") @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class SegmentReplicationRemoteStoreIT extends SegmentReplicationIT { @@ -49,7 +47,7 @@ protected Settings featureFlagSettings() { } @Before - public void setup() { + private void setup() { internalCluster().startClusterManagerOnlyNode(); Path absolutePath = randomRepoPath().toAbsolutePath(); assertAcked( diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 93d4d4bbfec8b..9938d11caca13 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1579,13 +1579,21 @@ public Tuple, ReplicationCheckpoint> getLatestSegme if (indexSettings.isSegRepEnabled() == false) { return null; } + + Tuple, ReplicationCheckpoint> nullSegmentInfosEmptyCheckpoint = new Tuple<>( + new GatedCloseable<>(null, () -> {}), + ReplicationCheckpoint.empty(shardId, getDefaultCodecName()) + ); + if (getEngineOrNull() == null) { - return new Tuple<>(new GatedCloseable<>(null, () -> {}), ReplicationCheckpoint.empty(shardId, getDefaultCodecName())); + return nullSegmentInfosEmptyCheckpoint; } // do not close the snapshot - caller will close it. - final GatedCloseable snapshot = getSegmentInfosSnapshot(); - return Optional.ofNullable(snapshot.get()).map(segmentInfos -> { - try { + GatedCloseable snapshot = null; + try { + snapshot = getSegmentInfosSnapshot(); + if (snapshot.get() != null) { + SegmentInfos segmentInfos = snapshot.get(); return new Tuple<>( snapshot, new ReplicationCheckpoint( @@ -1601,10 +1609,18 @@ public Tuple, ReplicationCheckpoint> getLatestSegme getEngine().config().getCodec().getName() ) ); - } catch (IOException e) { - throw new OpenSearchException("Error Fetching SegmentInfos and latest checkpoint", e); } - }).orElseGet(() -> new Tuple<>(new GatedCloseable<>(null, () -> {}), ReplicationCheckpoint.empty(shardId, getDefaultCodecName()))); + } catch (IOException | AlreadyClosedException e) { + logger.error("Error Fetching SegmentInfos and latest checkpoint", e); + if (snapshot != null) { + try { + snapshot.close(); + } catch (IOException ex) { + throw new OpenSearchException("Error Closing SegmentInfos Snapshot", e); + } + } + } + return nullSegmentInfosEmptyCheckpoint; } /** diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 1c2ddf43f8274..688f29fa1f4bf 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -282,7 +282,6 @@ public void testRefreshSuccessOnSecondAttempt() throws Exception { /** * Tests retry flow after snapshot and metadata files have been uploaded to remote store in the failed attempt. * Snapshot and metadata files created in failed attempt should not break retry. - * @throws Exception */ public void testRefreshSuccessAfterFailureInFirstAttemptAfterSnapshotAndMetadataUpload() throws Exception { int succeedOnAttempt = 1; From 9d9a143ff7a896bcfb116fb133f84718b73dec8b Mon Sep 17 00:00:00 2001 From: Ashish Date: Tue, 27 Jun 2023 15:16:21 +0530 Subject: [PATCH 008/186] [Remote Store] Add remote segment upload backpressure integ tests (#8197) Signed-off-by: Ashish Singh --- ...emoteStoreMockRepositoryIntegTestCase.java | 5 +- .../RemoteStoreBackpressureIT.java | 117 ++++++++++++++++-- .../snapshots/mockstore/MockRepository.java | 6 +- 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 2bcbf3f5b614d..f57c312aa2cd0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -70,7 +70,7 @@ protected void deleteRepo() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); } - protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { + protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation); // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the @@ -88,13 +88,14 @@ protected void setup(Path repoLocation, double ioFailureRate, String skipExcepti .put("max_failure_number", maxFailure) ); - internalCluster().startDataOnlyNodes(1); + String dataNodeName = internalCluster().startDataOnlyNodes(1).get(0); createIndex(INDEX_NAME); logger.info("--> Created index={}", INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); logger.info("--> Cluster is yellow with no initializing shards"); ensureGreen(INDEX_NAME); logger.info("--> Cluster is green"); + return dataNodeName; } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index c46eab6468c6b..64d5f06f061a9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -11,40 +11,111 @@ import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.opensearch.common.bytes.BytesArray; +import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.snapshots.mockstore.MockRepository; import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.opensearch.index.remote.RemoteRefreshSegmentPressureSettings.MIN_CONSECUTIVE_FAILURES_LIMIT; import static org.opensearch.index.remote.RemoteRefreshSegmentPressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreBackpressureIT extends AbstractRemoteStoreMockRepositoryIntegTestCase { + public void testWritesRejectedDueToConsecutiveFailureBreach() throws Exception { + // Here the doc size of the request remains same throughout the test. After initial indexing, all remote store interactions + // fail leading to consecutive failure limit getting exceeded and leading to rejections. + validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 10, ByteSizeUnit.KB.toIntBytes(1), 15, "failure_streak_count"); + } + + public void testWritesRejectedDueToBytesLagBreach() throws Exception { + // Initially indexing happens with doc size of 2 bytes, then all remote store interactions start failing. Now, the + // indexing happens with doc size of 1KB leading to bytes lag limit getting exceeded and leading to rejections. + validateBackpressure(ByteSizeUnit.BYTES.toIntBytes(2), 30, ByteSizeUnit.KB.toIntBytes(1), 15, "bytes_lag"); + } - public void testWritesRejected() { + public void testWritesRejectedDueToTimeLagBreach() throws Exception { + // Initially indexing happens with doc size of 1KB, then all remote store interactions start failing. Now, the + // indexing happens with doc size of 1 byte leading to time lag limit getting exceeded and leading to rejections. + validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 20, ByteSizeUnit.BYTES.toIntBytes(1), 15, "time_lag"); + } + + private void validateBackpressure( + int initialDocSize, + int initialDocsToIndex, + int onFailureDocSize, + int onFailureDocsToIndex, + String breachMode + ) throws Exception { Path location = randomRepoPath().toAbsolutePath(); - setup(location, 1d, "metadata", Long.MAX_VALUE); + String dataNodeName = setup(location, 0d, "metadata", Long.MAX_VALUE); - Settings request = Settings.builder().put(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), true).build(); + Settings request = Settings.builder() + .put(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), true) + .put(MIN_CONSECUTIVE_FAILURES_LIMIT.getKey(), 10) + .build(); ClusterUpdateSettingsResponse clusterUpdateResponse = client().admin() .cluster() .prepareUpdateSettings() .setPersistentSettings(request) .get(); assertEquals(clusterUpdateResponse.getPersistentSettings().get(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey()), "true"); + assertEquals(clusterUpdateResponse.getPersistentSettings().get(MIN_CONSECUTIVE_FAILURES_LIMIT.getKey()), "10"); logger.info("--> Indexing data"); + + String jsonString = generateString(initialDocSize); + BytesReference initialSource = new BytesArray(jsonString); + indexDocAndRefresh(initialSource, initialDocsToIndex); + + ((MockRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName).repository(REPOSITORY_NAME)) + .setRandomControlIOExceptionRate(1d); + + jsonString = generateString(onFailureDocSize); + BytesReference onFailureSource = new BytesArray(jsonString); OpenSearchRejectedExecutionException ex = assertThrows( OpenSearchRejectedExecutionException.class, - () -> indexData(randomIntBetween(10, 20), randomBoolean()) + () -> indexDocAndRefresh(onFailureSource, onFailureDocsToIndex) ); assertTrue(ex.getMessage().contains("rejected execution on primary shard")); + assertTrue(ex.getMessage().contains(breachMode)); + + RemoteRefreshSegmentTracker.Stats stats = stats(); + assertTrue(stats.bytesLag > 0); + assertTrue(stats.refreshTimeLagMs > 0); + assertTrue(stats.localRefreshNumber - stats.remoteRefreshNumber > 0); + assertTrue(stats.rejectionCount > 0); + + ((MockRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName).repository(REPOSITORY_NAME)) + .setRandomControlIOExceptionRate(0d); + + assertBusy(() -> { + RemoteRefreshSegmentTracker.Stats finalStats = stats(); + assertEquals(0, finalStats.bytesLag); + assertEquals(0, finalStats.refreshTimeLagMs); + assertEquals(0, finalStats.localRefreshNumber - finalStats.remoteRefreshNumber); + }, 30, TimeUnit.SECONDS); + + long rejectionCount = stats.rejectionCount; + stats = stats(); + indexDocAndRefresh(initialSource, initialDocsToIndex); + assertEquals(rejectionCount, stats.rejectionCount); + deleteRepo(); + } + + private RemoteRefreshSegmentTracker.Stats stats() { String shardId = "0"; RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); @@ -52,11 +123,37 @@ public void testWritesRejected() { .filter(stat -> indexShardId.equals(stat.getStats().shardId.toString())) .collect(Collectors.toList()); assertEquals(1, matches.size()); - RemoteRefreshSegmentTracker.Stats stats = matches.get(0).getStats(); - assertTrue(stats.bytesLag > 0); - assertTrue(stats.refreshTimeLagMs > 0); - assertTrue(stats.localRefreshNumber - stats.remoteRefreshNumber > 0); - assertTrue(stats.rejectionCount > 0); - deleteRepo(); + return matches.get(0).getStats(); + } + + private void indexDocAndRefresh(BytesReference source, int iterations) { + for (int i = 0; i < iterations; i++) { + client().prepareIndex(INDEX_NAME).setSource(source, XContentType.JSON).get(); + refresh(INDEX_NAME); + } + } + + /** + * Generates string of given sizeInBytes + * + * @param sizeInBytes size of the string + * @return the generated string + */ + private String generateString(int sizeInBytes) { + StringBuilder sb = new StringBuilder(); + sb.append("{"); + int i = 0; + // Based on local tests, 1 char is occupying 1 byte + while (sb.length() < sizeInBytes) { + String key = "field" + i; + String value = "value" + i; + sb.append("\"").append(key).append("\":\"").append(value).append("\","); + i++; + } + if (sb.length() > 1 && sb.charAt(sb.length() - 1) == ',') { + sb.setLength(sb.length() - 1); + } + sb.append("}"); + return sb.toString(); } } diff --git a/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java index fcaf9f6c900d3..7a7c4bd448c55 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java @@ -114,7 +114,7 @@ public long getFailureCount() { return failureCounter.get(); } - private final double randomControlIOExceptionRate; + private volatile double randomControlIOExceptionRate; private final double randomDataFileIOExceptionRate; @@ -246,6 +246,10 @@ public synchronized void unblock() { this.notifyAll(); } + public void setRandomControlIOExceptionRate(double randomControlIOExceptionRate) { + this.randomControlIOExceptionRate = randomControlIOExceptionRate; + } + public void blockOnDataFiles(boolean blocked) { blockOnDataFiles = blocked; } From 246b922fc1a15985254d38222fc6c62570f16cf3 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 28 Jun 2023 02:33:13 +1200 Subject: [PATCH 009/186] Bump `resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly (#8209) Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 + qa/wildfly/build.gradle | 37 +++++++++++-------- qa/wildfly/docker-compose.yml | 2 +- .../opensearch/wildfly/model/Employee.java | 4 -- .../RestHighLevelClientActivator.java | 12 +----- .../RestHighLevelClientEmployeeResource.java | 18 +++++---- .../RestHighLevelClientProducer.java | 3 +- .../RestHighLevelJacksonJsonProvider.java | 2 +- .../WEB-INF/jboss-deployment-structure.xml | 3 -- 10 files changed, 40 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bc63794e46d..50e1fe78daf5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257) - Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256) - [Upgrade] Lucene 9.7.0 release (#8272) +- Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209)) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 377421703b892..dd64569259c2d 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -64,3 +64,5 @@ jmh = 1.35 zstd = 1.5.5-3 jzlib = 1.1.3 + +resteasy = 6.2.4.Final diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle index a2a13165ca10c..391d2c78b489b 100644 --- a/qa/wildfly/build.gradle +++ b/qa/wildfly/build.gradle @@ -40,25 +40,32 @@ apply plugin: 'opensearch.internal-distribution-download' testFixtures.useFixture() dependencies { - providedCompile 'javax.enterprise:cdi-api:2.0' - providedCompile "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" - providedCompile 'jakarta.ws.rs:jakarta.ws.rs-api:2.1.3' - api('org.jboss.resteasy:resteasy-jackson2-provider:3.0.26.Final') { - exclude module: 'jackson-annotations' - exclude module: 'jackson-core' - exclude module: 'jackson-databind' - exclude module: 'jackson-jaxrs-json-provider' + providedCompile('jakarta.enterprise:jakarta.enterprise.cdi-api:4.0.1') { + exclude module: 'jakarta.annotation-api' + } + providedCompile 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0' + providedCompile "org.jboss.resteasy:resteasy-core:${versions.resteasy}" + providedCompile "org.jboss.resteasy:resteasy-core-spi:${versions.resteasy}" + api("org.jboss.resteasy:resteasy-jackson2-provider:${versions.resteasy}") { + exclude module: 'jakarta.activation-api' + exclude group: 'com.fasterxml.jackson' + exclude group: 'com.fasterxml.jackson.core' + exclude group: 'com.fasterxml.jackson.dataformat' + exclude group: 'com.fasterxml.jackson.module' } api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" - api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}" - api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:${versions.jackson}" - api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + api "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-base:${versions.jackson}" + api "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider:${versions.jackson}" + api "com.github.fge:json-patch:1.9" api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" - api project(path: ':client:rest-high-level') - testImplementation project(':test:framework') + api(project(path: ':client:rest-high-level')) { + exclude module: 'jakarta.annotation-api' + } + testImplementation(project(':test:framework')) { + exclude module: 'jakarta.annotation-api' + } } war { diff --git a/qa/wildfly/docker-compose.yml b/qa/wildfly/docker-compose.yml index 96f168ba3505c..b0f1609f01e72 100644 --- a/qa/wildfly/docker-compose.yml +++ b/qa/wildfly/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.7' services: wildfly: - image: jboss/wildfly:18.0.1.Final + image: quay.io/wildfly/wildfly:28.0.1.Final-jdk11 environment: JAVA_OPTS: -Dopensearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true volumes: diff --git a/qa/wildfly/src/main/java/org/opensearch/wildfly/model/Employee.java b/qa/wildfly/src/main/java/org/opensearch/wildfly/model/Employee.java index 5cf40ce941636..8f8d6635d568d 100644 --- a/qa/wildfly/src/main/java/org/opensearch/wildfly/model/Employee.java +++ b/qa/wildfly/src/main/java/org/opensearch/wildfly/model/Employee.java @@ -34,12 +34,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; -import javax.ws.rs.Consumes; -import javax.ws.rs.core.MediaType; - import java.util.List; -@Consumes(MediaType.APPLICATION_JSON) public class Employee { @JsonProperty(value = "first_name") diff --git a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientActivator.java b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientActivator.java index bf91346933e02..14b8cd6730531 100644 --- a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientActivator.java +++ b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientActivator.java @@ -32,18 +32,10 @@ package org.opensearch.wildfly.transport; -import javax.ws.rs.ApplicationPath; -import javax.ws.rs.core.Application; - -import java.util.Collections; -import java.util.Set; +import jakarta.ws.rs.ApplicationPath; +import jakarta.ws.rs.core.Application; @ApplicationPath("/transport") public class RestHighLevelClientActivator extends Application { - @Override - public Set> getClasses() { - return Collections.singleton(RestHighLevelClientEmployeeResource.class); - } - } diff --git a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientEmployeeResource.java b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientEmployeeResource.java index 036d782f6f5e8..432b40367c978 100644 --- a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientEmployeeResource.java +++ b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientEmployeeResource.java @@ -32,6 +32,15 @@ package org.opensearch.wildfly.transport; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexRequest; @@ -41,14 +50,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.wildfly.model.Employee; -import javax.inject.Inject; -import javax.ws.rs.GET; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -88,6 +89,7 @@ public Response getEmployeeById(final @PathParam("id") Long id) throws IOExcepti @PUT @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public Response putEmployeeById(final @PathParam("id") Long id, final Employee employee) throws URISyntaxException, IOException { Objects.requireNonNull(id); diff --git a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientProducer.java b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientProducer.java index 2b1abe45f7723..490ecd214c3f3 100644 --- a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientProducer.java +++ b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientProducer.java @@ -32,14 +32,13 @@ package org.opensearch.wildfly.transport; +import jakarta.enterprise.inject.Produces; import org.apache.hc.core5.http.HttpHost; import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.PathUtils; -import javax.enterprise.inject.Produces; - import java.net.URISyntaxException; import java.nio.file.Path; diff --git a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelJacksonJsonProvider.java b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelJacksonJsonProvider.java index 604d975f53280..7989f0351daef 100644 --- a/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelJacksonJsonProvider.java +++ b/qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelJacksonJsonProvider.java @@ -34,7 +34,7 @@ import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.ext.Provider; @Provider public class RestHighLevelJacksonJsonProvider extends ResteasyJackson2Provider { diff --git a/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml b/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml index 7191bfe1268aa..a08090100989a 100644 --- a/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml +++ b/qa/wildfly/src/main/webapp/WEB-INF/jboss-deployment-structure.xml @@ -1,9 +1,6 @@ - - - From f936b896ced3ec33f7116f52f45f1df6fd61fc52 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 27 Jun 2023 13:22:54 -0400 Subject: [PATCH 010/186] Fix Apache Lucene verson for 2.9.0 release (#8290) Signed-off-by: Andriy Redko --- libs/core/src/main/java/org/opensearch/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index c399f7f3e1b1c..8c0e247bc3169 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -89,7 +89,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0); - public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_6_0); + public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version CURRENT = V_3_0_0; From 317dd03ed83110a5c4ab665f3682a22a7c1c864a Mon Sep 17 00:00:00 2001 From: Harish Bhakuni Date: Tue, 27 Jun 2023 11:01:58 -0700 Subject: [PATCH 011/186] [Snapshot Interop] Add Changes in Restore Snapshot Flow for remote store Interoperability (#6788) * [Snapshot Interop] Add Changes in Restore Snapshot Flow for remote store interoperability. --------- Signed-off-by: Harish Bhakuni Co-authored-by: Harish Bhakuni --- .../rest-api-spec/api/snapshot.restore.json | 4 + .../snapshots/RestoreSnapshotIT.java | 513 ++++++++++++++++++ .../restore/RestoreSnapshotRequest.java | 42 +- .../RestoreSnapshotRequestBuilder.java | 8 + .../cluster/routing/RecoverySource.java | 50 +- .../opensearch/index/shard/IndexShard.java | 199 +++++-- .../shard/RemoteStoreRefreshListener.java | 15 +- .../opensearch/index/shard/StoreRecovery.java | 98 +++- .../store/RemoteSegmentStoreDirectory.java | 19 + .../RemoteSegmentStoreDirectoryFactory.java | 5 + .../metadata/RemoteSegmentMetadata.java | 11 +- .../index/translog/RemoteFsTranslog.java | 14 + .../transfer/TranslogTransferManager.java | 5 + .../cluster/IndicesClusterStateService.java | 15 +- .../repositories/FilterRepository.java | 10 + .../opensearch/repositories/Repository.java | 17 + .../blobstore/BlobStoreRepository.java | 10 + .../opensearch/snapshots/RestoreService.java | 15 +- .../restore/RestoreSnapshotRequestTests.java | 4 + .../cluster/routing/RecoverySourceTests.java | 24 + .../index/shard/IndexShardTests.java | 77 ++- .../RemoteSegmentStoreDirectoryTests.java | 18 +- .../RemoteSegmentMetadataHandlerTests.java | 4 + .../RepositoriesServiceTests.java | 10 + .../blobstore/BlobStoreRepositoryTests.java | 92 +++- .../index/shard/IndexShardTestCase.java | 32 +- .../blobstore/BlobStoreTestUtil.java | 31 +- .../AbstractSnapshotIntegTestCase.java | 48 +- 28 files changed, 1303 insertions(+), 87 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json index 87ab8117ec489..07148c7d261f4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json @@ -42,6 +42,10 @@ "type":"boolean", "description":"Should this request wait until the operation has completed before returning", "default":false + }, + "source_remote_store_repository": { + "type":"string", + "description":"Remote Store Repository of Remote Store Indices" } }, "body":{ diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index 8be14d1188db8..9f492bbaee01a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -33,29 +33,44 @@ package org.opensearch.snapshots; import org.opensearch.action.ActionFuture; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.admin.indices.get.GetIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse; +import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.client.Client; +import org.opensearch.client.Requests; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.index.IndexSettings; import org.opensearch.indices.InvalidIndexNameException; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestStatus; +import org.opensearch.test.InternalTestCluster; +import java.io.IOException; import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Arrays; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -70,6 +85,8 @@ import static org.hamcrest.Matchers.nullValue; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.index.query.QueryBuilders.matchQuery; @@ -81,6 +98,10 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows; public class RestoreSnapshotIT extends AbstractSnapshotIntegTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(FeatureFlags.REMOTE_STORE, "true").build(); + } public void testParallelRestoreOperations() { String indexName1 = "testindex1"; @@ -152,6 +173,498 @@ public void testParallelRestoreOperations() { assertThat(client.prepareGet(restoredIndexName2, docId2).get().isExists(), equalTo(true)); } + public void testRestoreRemoteStoreIndicesWithRemoteTranslog() throws IOException, ExecutionException, InterruptedException { + testRestoreOperationsShallowCopyEnabled(true); + } + + public void testRestoreRemoteStoreIndicesWithoutRemoteTranslog() throws IOException, ExecutionException, InterruptedException { + testRestoreOperationsShallowCopyEnabled(false); + } + + public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnabled) throws IOException, ExecutionException, + InterruptedException { + internalCluster().startClusterManagerOnlyNode(); + final String primaryNode = internalCluster().startNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + logger.info("Remote Store Repo Path [{}]", absolutePath2); + String restoredIndexName1 = indexName1 + "-restored"; + String restoredIndexName1Seg = indexName1 + "-restored-seg"; + String restoredIndexName1Doc = indexName1 + "-restored-doc"; + String restoredIndexName2 = indexName2 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + createRepository(remoteStoreRepoName, "fs", absolutePath2); + + Client client = client(); + Settings indexSettings = getIndexSettings(true, remoteTranslogEnabled, remoteStoreRepoName, 1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(false, false, null, 1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + final String secondNode = internalCluster().startNode(); + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + CreateSnapshotResponse createSnapshotResponse2 = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse2.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); + ensureGreen(indexName1); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .setRenamePattern(indexName2) + .setRenameReplacement(restoredIndexName2) + .get(); + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1, restoredIndexName2); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1))); + ensureRed(restoredIndexName1); + assertAcked(client().admin().indices().prepareClose(restoredIndexName1)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); + ensureYellowAndNoInitializingShards(restoredIndexName1); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + + // restore index as seg rep enabled with remote store and remote translog disabled + RestoreSnapshotResponse restoreSnapshotResponse3 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1Seg) + .get(); + assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1Seg); + + GetIndexResponse getIndexResponse = client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true)) + .get(); + indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg); + assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); + assertNull(indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY, null)); + assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); + assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1Seg, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1Seg); + assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1 + 2); + + // restore index as doc rep based from shallow copy snapshot + RestoreSnapshotResponse restoreSnapshotResponse4 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings( + IndexMetadata.SETTING_REMOTE_STORE_ENABLED, + IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, + IndexMetadata.SETTING_REPLICATION_TYPE + ) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1Doc) + .get(); + assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1Doc); + + getIndexResponse = client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true)) + .get(); + indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc); + assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); + assertNull(indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY, null)); + assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); + assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1Doc, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1Doc); + assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2); + } + + public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + logger.info("Remote Store Repo Path [{}]", absolutePath2); + String restoredIndexName2 = indexName2 + "-restored"; + + boolean enableShallowCopy = randomBoolean(); + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, enableShallowCopy)); + createRepository(remoteStoreRepoName, "fs", absolutePath2); + + Client client = client(); + Settings indexSettings = getIndexSettings(true, randomBoolean(), remoteStoreRepoName, 1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(false, false, null, 1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + final String secondNode = internalCluster().startNode(); + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + CreateSnapshotResponse createSnapshotResponse2 = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse2.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); + ensureGreen(indexName1); + + assertAcked(client().admin().indices().prepareClose(indexName1)); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .setRenamePattern(indexName2) + .setRenameReplacement(restoredIndexName2) + .get(); + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); + ensureGreen(indexName1, restoredIndexName2); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(indexName1))); + ensureRed(indexName1); + assertAcked(client().admin().indices().prepareClose(indexName1)); + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture()); + ensureYellowAndNoInitializingShards(indexName1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 2); + } + + public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException { + internalCluster().startClusterManagerOnlyNode(); + final String primaryNode = internalCluster().startNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + String remoteStoreRepo2Name = "test-rs-repo-2" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + Path absolutePath3 = randomRepoPath().toAbsolutePath(); + String restoredIndexName1 = indexName1 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + createRepository(remoteStoreRepoName, "fs", absolutePath2); + createRepository(remoteStoreRepo2Name, "fs", absolutePath3); + + Client client = client(); + Settings indexSettings = getIndexSettings(true, true, remoteStoreRepoName, 1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(false, false, null, 1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + final String secondNode = internalCluster().startNode(); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + Settings remoteStoreIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo2Name) + .build(); + // restore index as a remote store index with different remote store repo + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreIndexSettings) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1))); + assertAcked(client().admin().indices().prepareClose(restoredIndexName1)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); + ensureYellowAndNoInitializingShards(restoredIndexName1); + ensureGreen(restoredIndexName1); + // indexing some new docs and validating + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + + private Settings.Builder getIndexSettings( + boolean enableRemoteStore, + boolean enableRemoteTranslog, + String remoteStoreRepo, + int numOfShards, + int numOfReplicas + ) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas); + if (enableRemoteStore) { + settingsBuilder.put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); + } + if (enableRemoteTranslog) { + settingsBuilder.put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) + .build(); + } + return settingsBuilder; + } + + public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionException, InterruptedException { + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + Path absolutePath3 = randomRepoPath().toAbsolutePath(); + String[] pathTokens = absolutePath1.toString().split("/"); + String basePath = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location = PathUtils.get(String.join("/", pathTokens)); + pathTokens = absolutePath2.toString().split("/"); + String basePath2 = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location2 = PathUtils.get(String.join("/", pathTokens)); + logger.info("Path 1 [{}]", absolutePath1); + logger.info("Path 2 [{}]", absolutePath2); + logger.info("Path 3 [{}]", absolutePath3); + String restoredIndexName1 = indexName1 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(location, basePath, true)); + createRepository(remoteStoreRepoName, "fs", absolutePath3); + + Client client = client(); + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepoName) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + createIndex(indexName1, indexSettings); + + int numDocsInIndex1 = randomIntBetween(2, 5); + indexDocuments(client, indexName1, numDocsInIndex1); + + ensureGreen(indexName1); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + createRepository(remoteStoreRepoName, "fs", absolutePath2); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + + assertTrue(restoreSnapshotResponse.getRestoreInfo().failedShards() > 0); + + ensureRed(restoredIndexName1); + + client().admin().indices().close(Requests.closeIndexRequest(restoredIndexName1)).get(); + createRepository(remoteStoreRepoNameUpdated, "fs", absolutePath3); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .setSourceRemoteStoreRepository(remoteStoreRepoNameUpdated) + .get(); + + assertTrue(restoreSnapshotResponse2.getRestoreInfo().failedShards() == 0); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + + private void indexDocuments(Client client, String indexName, int numOfDocs) { + indexDocuments(client, indexName, 0, numOfDocs); + } + + private void indexDocuments(Client client, String indexName, int fromId, int toId) { + for (int i = fromId; i < toId; i++) { + String id = Integer.toString(i); + client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + } + client.admin().indices().prepareFlush(indexName).get(); + } + + private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { + for (int i = 0; i < numOfDocs; i++) { + String id = Integer.toString(i); + logger.info("checking for index " + indexName + " with docId" + id); + assertTrue("doc with id" + id + " is not present for index " + indexName, client.prepareGet(indexName, id).get().isExists()); + } + } + public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception { String indexName1 = "testindex1"; String indexName2 = "testindex2"; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index e127b44116b7e..7ff16fcf6fc03 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -113,6 +113,8 @@ private static StorageType fromString(String string) { private Settings indexSettings = EMPTY_SETTINGS; private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY; private StorageType storageType = StorageType.LOCAL; + @Nullable + private String sourceRemoteStoreRepository = null; @Nullable // if any snapshot UUID will do private String snapshotUuid; @@ -148,6 +150,9 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_7_0)) { storageType = in.readEnum(StorageType.class); } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + sourceRemoteStoreRepository = in.readOptionalString(); + } } @Override @@ -169,6 +174,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_7_0)) { out.writeEnum(storageType); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(sourceRemoteStoreRepository); + } } @Override @@ -521,6 +529,25 @@ public StorageType storageType() { return storageType; } + /** + * Sets Source Remote Store Repository for all the restored indices + * + * @param sourceRemoteStoreRepository name of the remote store repository that should be used for all restored indices. + */ + public RestoreSnapshotRequest setSourceRemoteStoreRepository(String sourceRemoteStoreRepository) { + this.sourceRemoteStoreRepository = sourceRemoteStoreRepository; + return this; + } + + /** + * Returns Source Remote Store Repository for all the restored indices + * + * @return source Remote Store Repository + */ + public String getSourceRemoteStoreRepository() { + return sourceRemoteStoreRepository; + } + /** * Parses restore definition * @@ -586,6 +613,12 @@ public RestoreSnapshotRequest source(Map source) { throw new IllegalArgumentException("malformed storage_type"); } + } else if (name.equals("source_remote_store_repository")) { + if (entry.getValue() instanceof String) { + setSourceRemoteStoreRepository((String) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed source_remote_store_repository"); + } } else { if (IndicesOptions.isIndicesOptions(name) == false) { throw new IllegalArgumentException("Unknown parameter " + name); @@ -631,6 +664,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (storageType != null) { storageType.toXContent(builder); } + if (sourceRemoteStoreRepository != null) { + builder.field("source_remote_store_repository", sourceRemoteStoreRepository); + } builder.endObject(); return builder; } @@ -658,7 +694,8 @@ public boolean equals(Object o) { && Objects.equals(indexSettings, that.indexSettings) && Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings) && Objects.equals(snapshotUuid, that.snapshotUuid) - && Objects.equals(storageType, that.storageType); + && Objects.equals(storageType, that.storageType) + && Objects.equals(sourceRemoteStoreRepository, that.sourceRemoteStoreRepository); } @Override @@ -675,7 +712,8 @@ public int hashCode() { includeAliases, indexSettings, snapshotUuid, - storageType + storageType, + sourceRemoteStoreRepository ); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(ignoreIndexSettings); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 0104637a00035..d9cca536d1c41 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -256,4 +256,12 @@ public RestoreSnapshotRequestBuilder setStorageType(RestoreSnapshotRequest.Stora request.storageType(storageType); return this; } + + /** + * Sets the source remote store repository name + */ + public RestoreSnapshotRequestBuilder setSourceRemoteStoreRepository(String repositoryName) { + request.setSourceRemoteStoreRepository(repositoryName); + return this; + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index ef86eb31e2817..1af4fbe8ffb45 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; @@ -257,9 +258,11 @@ public static class SnapshotRecoverySource extends RecoverySource { private final IndexId index; private final Version version; private final boolean isSearchableSnapshot; + private final boolean remoteStoreIndexShallowCopy; + private final String sourceRemoteStoreRepository; public SnapshotRecoverySource(String restoreUUID, Snapshot snapshot, Version version, IndexId indexId) { - this(restoreUUID, snapshot, version, indexId, false); + this(restoreUUID, snapshot, version, indexId, false, false, null); } public SnapshotRecoverySource( @@ -267,13 +270,17 @@ public SnapshotRecoverySource( Snapshot snapshot, Version version, IndexId indexId, - boolean isSearchableSnapshot + boolean isSearchableSnapshot, + boolean remoteStoreIndexShallowCopy, + @Nullable String sourceRemoteStoreRepository ) { this.restoreUUID = restoreUUID; this.snapshot = Objects.requireNonNull(snapshot); this.version = Objects.requireNonNull(version); this.index = Objects.requireNonNull(indexId); this.isSearchableSnapshot = isSearchableSnapshot; + this.remoteStoreIndexShallowCopy = remoteStoreIndexShallowCopy; + this.sourceRemoteStoreRepository = sourceRemoteStoreRepository; } SnapshotRecoverySource(StreamInput in) throws IOException { @@ -286,6 +293,13 @@ public SnapshotRecoverySource( } else { isSearchableSnapshot = false; } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + remoteStoreIndexShallowCopy = in.readBoolean(); + sourceRemoteStoreRepository = in.readOptionalString(); + } else { + remoteStoreIndexShallowCopy = false; + sourceRemoteStoreRepository = null; + } } public String restoreUUID() { @@ -314,6 +328,14 @@ public boolean isSearchableSnapshot() { return isSearchableSnapshot; } + public String sourceRemoteStoreRepository() { + return sourceRemoteStoreRepository; + } + + public boolean remoteStoreIndexShallowCopy() { + return remoteStoreIndexShallowCopy; + } + @Override protected void writeAdditionalFields(StreamOutput out) throws IOException { out.writeString(restoreUUID); @@ -323,6 +345,10 @@ protected void writeAdditionalFields(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_7_0)) { out.writeBoolean(isSearchableSnapshot); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeBoolean(remoteStoreIndexShallowCopy); + out.writeOptionalString(sourceRemoteStoreRepository); + } } @Override @@ -337,7 +363,9 @@ public void addAdditionalFields(XContentBuilder builder, ToXContent.Params param .field("version", version.toString()) .field("index", index.getName()) .field("restoreUUID", restoreUUID) - .field("isSearchableSnapshot", isSearchableSnapshot); + .field("isSearchableSnapshot", isSearchableSnapshot) + .field("remoteStoreIndexShallowCopy", remoteStoreIndexShallowCopy) + .field("sourceRemoteStoreRepository", sourceRemoteStoreRepository); } @Override @@ -359,12 +387,24 @@ public boolean equals(Object o) { && snapshot.equals(that.snapshot) && index.equals(that.index) && version.equals(that.version) - && isSearchableSnapshot == that.isSearchableSnapshot; + && isSearchableSnapshot == that.isSearchableSnapshot + && remoteStoreIndexShallowCopy == that.remoteStoreIndexShallowCopy + && sourceRemoteStoreRepository != null + ? sourceRemoteStoreRepository.equals(that.sourceRemoteStoreRepository) + : that.sourceRemoteStoreRepository == null; } @Override public int hashCode() { - return Objects.hash(restoreUUID, snapshot, index, version, isSearchableSnapshot); + return Objects.hash( + restoreUUID, + snapshot, + index, + version, + isSearchableSnapshot, + remoteStoreIndexShallowCopy, + sourceRemoteStoreRepository + ); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 9938d11caca13..d89d51c713d70 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -44,6 +44,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ReferenceManager; @@ -1495,7 +1496,7 @@ public GatedCloseable acquireLastIndexCommitAndRefresh(boolean flus * @throws IOException if there is some failure in acquiring lock in remote store. */ public void acquireLockOnCommitData(String snapshotId, long primaryTerm, long generation) throws IOException { - RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = getRemoteSegmentDirectoryForShard(); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = getRemoteDirectory(); remoteSegmentStoreDirectory.acquireLock(primaryTerm, generation, snapshotId); } @@ -1507,20 +1508,10 @@ public void acquireLockOnCommitData(String snapshotId, long primaryTerm, long ge * @throws IOException if there is some failure in releasing lock in remote store. */ public void releaseLockOnCommitData(String snapshotId, long primaryTerm, long generation) throws IOException { - RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = getRemoteSegmentDirectoryForShard(); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = getRemoteDirectory(); remoteSegmentStoreDirectory.releaseLock(primaryTerm, generation, snapshotId); } - private RemoteSegmentStoreDirectory getRemoteSegmentDirectoryForShard() { - FilterDirectory remoteStoreDirectory = (FilterDirectory) remoteStore.directory(); - assert remoteStoreDirectory.getDelegate() instanceof FilterDirectory - : "Store.directory is not enclosing an instance of FilterDirectory"; - FilterDirectory byteSizeCachingStoreDirectory = (FilterDirectory) remoteStoreDirectory.getDelegate(); - final Directory remoteDirectory = byteSizeCachingStoreDirectory.getDelegate(); - assert remoteDirectory instanceof RemoteSegmentStoreDirectory : "remoteDirectory is not an instance of RemoteSegmentStoreDirectory"; - return ((RemoteSegmentStoreDirectory) remoteDirectory); - } - public Optional getReplicationEngine() { if (getEngine() instanceof NRTReplicationEngine) { return Optional.of((NRTReplicationEngine) getEngine()); @@ -2290,7 +2281,24 @@ public void openEngineAndSkipTranslogRecovery() throws IOException { getEngine().translogManager().skipTranslogRecovery(); } + public void openEngineAndSkipTranslogRecoveryFromSnapshot() throws IOException { + assert routingEntry().recoverySource().getType() == RecoverySource.Type.SNAPSHOT : "not a snapshot recovery [" + + routingEntry() + + "]"; + recoveryState.validateCurrentStage(RecoveryState.Stage.INDEX); + maybeCheckIndex(); + recoveryState.setStage(RecoveryState.Stage.TRANSLOG); + recoveryState.validateCurrentStage(RecoveryState.Stage.TRANSLOG); + loadGlobalCheckpointToReplicationTracker(); + innerOpenEngineAndTranslog(replicationTracker, false); + getEngine().translogManager().skipTranslogRecovery(); + } + private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) throws IOException { + innerOpenEngineAndTranslog(globalCheckpointSupplier, true); + } + + private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier, boolean syncFromRemote) throws IOException { assert Thread.holdsLock(mutex) == false : "opening engine under mutex"; if (state != IndexShardState.RECOVERING) { throw new IndexShardNotRecoveringException(shardId, state); @@ -2309,11 +2317,20 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) t synchronized (engineMutex) { assert currentEngineReference.get() == null : "engine is running"; verifyNotClosed(); - if (indexSettings.isRemoteStoreEnabled()) { + if (indexSettings.isRemoteStoreEnabled() && syncFromRemote) { syncSegmentsFromRemoteSegmentStore(false, true, true); } if (indexSettings.isRemoteTranslogStoreEnabled() && shardRouting.primary()) { - syncRemoteTranslogAndUpdateGlobalCheckpoint(); + if (syncFromRemote) { + syncRemoteTranslogAndUpdateGlobalCheckpoint(); + } else { + // we will enter this block when we do not want to recover from remote translog. + // currently only during snapshot restore, we are coming into this block. + // here, as while initiliazing remote translog we cannot skip downloading translog files, + // so before that step, we are deleting the translog files present in remote store. + deleteTranslogFilesFromRemoteTranslog(); + + } } // we must create a new engine under mutex (see IndexShard#snapshotStoreMetadata). final Engine newEngine = engineFactory.newReadWriteEngine(config); @@ -2605,6 +2622,22 @@ public void restoreFromRemoteStore(ActionListener listener) { storeRecovery.recoverFromRemoteStore(this, listener); } + public void restoreFromSnapshotAndRemoteStore( + Repository repository, + RepositoriesService repositoriesService, + ActionListener listener + ) { + try { + assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard"; + assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + + recoveryState.getRecoverySource(); + StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); + storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, repositoriesService, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + public void restoreFromRepository(Repository repository, ActionListener listener) { try { assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard"; @@ -3418,6 +3451,15 @@ public void startRecovery( final SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) recoveryState.getRecoverySource(); if (recoverySource.isSearchableSnapshot()) { executeRecovery("from snapshot (remote)", recoveryState, recoveryListener, this::recoverFromStore); + } else if (recoverySource.remoteStoreIndexShallowCopy()) { + final String repo = recoverySource.snapshot().getRepository(); + executeRecovery( + "from snapshot and remote store", + recoveryState, + recoveryListener, + l -> restoreFromSnapshotAndRemoteStore(repositoriesService.repository(repo), repositoriesService, l) + ); + // indicesService.indexService(shardRouting.shardId().getIndex()).addMetadataListener(); } else { final String repo = recoverySource.snapshot().getRepository(); executeRecovery( @@ -4536,6 +4578,13 @@ private void syncRemoteTranslogAndUpdateGlobalCheckpoint() throws IOException { loadGlobalCheckpointToReplicationTracker(); } + public void deleteTranslogFilesFromRemoteTranslog() throws IOException { + TranslogFactory translogFactory = translogFactorySupplier.apply(indexSettings, shardRouting); + assert translogFactory instanceof RemoteBlobStoreInternalTranslogFactory; + Repository repository = ((RemoteBlobStoreInternalTranslogFactory) translogFactory).getRepository(); + RemoteFsTranslog.cleanup(repository, shardId, getThreadPool()); + } + public void syncTranslogFilesFromRemoteTranslog() throws IOException { TranslogFactory translogFactory = translogFactorySupplier.apply(indexSettings, shardRouting); assert translogFactory instanceof RemoteBlobStoreInternalTranslogFactory; @@ -4558,12 +4607,11 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re // We need to call RemoteSegmentStoreDirectory.init() in order to get latest metadata of the files that // are uploaded to the remote segment store. RemoteSegmentMetadata remoteSegmentMetadata = remoteDirectory.init(); - Map uploadedSegments = ((RemoteSegmentStoreDirectory) remoteDirectory) + + Map uploadedSegments = remoteDirectory .getSegmentsUploadedToRemoteStore(); store.incRef(); remoteStore.incRef(); - List downloadedSegments = new ArrayList<>(); - List skippedSegments = new ArrayList<>(); try { final Directory storeDirectory; if (recoveryState.getStage() == RecoveryState.Stage.INDEX) { @@ -4580,18 +4628,7 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re storeDirectory = store.directory(); } Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); - for (String file : uploadedSegments.keySet()) { - long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); - if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - if (localSegmentFiles.contains(file)) { - storeDirectory.deleteFile(file); - } - storeDirectory.copyFrom(remoteDirectory, file, file, IOContext.DEFAULT); - downloadedSegments.add(file); - } else { - skippedSegments.add(file); - } - } + copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal); if (refreshLevelSegmentSync && remoteSegmentMetadata != null) { try ( @@ -4637,13 +4674,113 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re } catch (IOException e) { throw new IndexShardRecoveryException(shardId, "Exception while copying segment files from remote segment store", e); } finally { - logger.info("Downloaded segments: {}", downloadedSegments); - logger.info("Skipped download for segments: {}", skippedSegments); store.decRef(); remoteStore.decRef(); } } + /** + * Downloads segments from given remote segment store for a specific commit. + * @param overrideLocal flag to override local segment files with those in remote store + * @param sourceRemoteDirectory RemoteSegmentDirectory Instance from which we need to sync segments + * @param primaryTerm Primary Term for shard at the time of commit operation for which we are syncing segments + * @param commitGeneration commit generation at the time of commit operation for which we are syncing segments + * @throws IOException if exception occurs while reading segments from remote store + */ + public void syncSegmentsFromGivenRemoteSegmentStore( + boolean overrideLocal, + RemoteSegmentStoreDirectory sourceRemoteDirectory, + long primaryTerm, + long commitGeneration + ) throws IOException { + logger.info("Downloading segments from given remote segment store"); + RemoteSegmentStoreDirectory remoteDirectory = null; + if (remoteStore != null) { + remoteDirectory = getRemoteDirectory(); + remoteDirectory.init(); + remoteStore.incRef(); + } + Map uploadedSegments = sourceRemoteDirectory + .initializeToSpecificCommit(primaryTerm, commitGeneration) + .getMetadata(); + final Directory storeDirectory = store.directory(); + store.incRef(); + + try { + String segmentsNFile = copySegmentFiles( + storeDirectory, + sourceRemoteDirectory, + remoteDirectory, + uploadedSegments, + overrideLocal + ); + if (segmentsNFile != null) { + try ( + ChecksumIndexInput indexInput = new BufferedChecksumIndexInput( + storeDirectory.openInput(segmentsNFile, IOContext.DEFAULT) + ) + ) { + SegmentInfos infosSnapshot = SegmentInfos.readCommit(store.directory(), indexInput, commitGeneration); + long processedLocalCheckpoint = Long.parseLong(infosSnapshot.getUserData().get(LOCAL_CHECKPOINT_KEY)); + if (remoteStore != null) { + store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint); + } else { + store.directory().sync(infosSnapshot.files(true)); + store.directory().syncMetaData(); + } + } + } + } catch (IOException e) { + throw new IndexShardRecoveryException(shardId, "Exception while copying segment files from remote segment store", e); + } finally { + store.decRef(); + if (remoteStore != null) { + remoteStore.decRef(); + } + } + } + + private String copySegmentFiles( + Directory storeDirectory, + RemoteSegmentStoreDirectory sourceRemoteDirectory, + RemoteSegmentStoreDirectory targetRemoteDirectory, + Map uploadedSegments, + boolean overrideLocal + ) throws IOException { + List downloadedSegments = new ArrayList<>(); + List skippedSegments = new ArrayList<>(); + String segmentNFile = null; + try { + Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); + if (overrideLocal) { + for (String file : localSegmentFiles) { + storeDirectory.deleteFile(file); + } + } + for (String file : uploadedSegments.keySet()) { + long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); + if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { + storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); + storeDirectory.sync(Collections.singleton(file)); + downloadedSegments.add(file); + } else { + skippedSegments.add(file); + } + if (targetRemoteDirectory != null) { + targetRemoteDirectory.copyFrom(storeDirectory, file, file, IOContext.DEFAULT); + } + if (file.startsWith(IndexFileNames.SEGMENTS)) { + assert segmentNFile == null : "There should be only one SegmentInfosSnapshot file"; + segmentNFile = file; + } + } + } finally { + logger.info("Downloaded segments here: {}", downloadedSegments); + logger.info("Skipped download for segments here: {}", skippedSegments); + } + return segmentNFile; + } + private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { if (checksum == CodecUtil.retrieveChecksum(indexInput)) { diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index d7f7373e83bd0..7cfaaafcadd39 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -29,6 +29,7 @@ import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.threadpool.Scheduler; @@ -71,6 +72,8 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres */ private static final int REMOTE_REFRESH_RETRY_MAX_INTERVAL_MILLIS = 10_000; + private static final int INVALID_PRIMARY_TERM = -1; + /** * Exponential back off policy with max retry interval. */ @@ -118,15 +121,18 @@ public RemoteStoreRefreshListener( this.storeDirectory = indexShard.store().directory(); this.remoteDirectory = (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()) .getDelegate()).getDelegate(); - this.primaryTerm = indexShard.getOperationPrimaryTerm(); localSegmentChecksumMap = new HashMap<>(); + RemoteSegmentMetadata remoteSegmentMetadata = null; if (indexShard.routingEntry().primary()) { try { - this.remoteDirectory.init(); + remoteSegmentMetadata = this.remoteDirectory.init(); } catch (IOException e) { logger.error("Exception while initialising RemoteSegmentStoreDirectory", e); } } + // initializing primary term with the primary term of latest metadata in remote store. + // if no metadata is present, this value will be initilized with -1. + this.primaryTerm = remoteSegmentMetadata != null ? remoteSegmentMetadata.getPrimaryTerm() : INVALID_PRIMARY_TERM; this.segmentTracker = segmentTracker; resetBackOffDelayIterator(); this.checkpointPublisher = checkpointPublisher; @@ -163,8 +169,9 @@ public void beforeRefresh() throws IOException {} */ @Override public void afterRefresh(boolean didRefresh) { - - if (didRefresh || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty()) { + if (this.primaryTerm != indexShard.getOperationPrimaryTerm() + || didRefresh + || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty()) { updateLocalRefreshTimeAndSeqNo(); try { indexShard.getThreadPool().executor(ThreadPool.Names.REMOTE_REFRESH).submit(() -> syncSegments(false)).get(); diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index a81dc96ff1145..119524e8caf8a 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -59,14 +59,19 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.snapshots.IndexShardRestoreFailedException; +import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.repositories.IndexId; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import java.io.IOException; +import java.nio.channels.FileChannel; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -347,6 +352,72 @@ void recoverFromRepository(final IndexShard indexShard, Repository repository, A } } + void recoverFromSnapshotAndRemoteStore( + final IndexShard indexShard, + Repository repository, + RepositoriesService repositoriesService, + ActionListener listener + ) { + try { + if (canRecover(indexShard)) { + indexShard.preRecovery(); + RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType(); + assert recoveryType == RecoverySource.Type.SNAPSHOT : "expected snapshot recovery type: " + recoveryType; + SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource(); + final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog(); + translogState.totalOperations(0); + translogState.totalOperationsOnStart(0); + indexShard.prepareForIndexRecovery(); + + RemoteStoreShardShallowCopySnapshot shallowCopyShardMetadata = repository.getRemoteStoreShallowCopyShardMetadata( + recoverySource.snapshot().getSnapshotId(), + recoverySource.index(), + shardId + ); + + long primaryTerm = shallowCopyShardMetadata.getPrimaryTerm(); + long commitGeneration = shallowCopyShardMetadata.getCommitGeneration(); + String indexUUID = shallowCopyShardMetadata.getIndexUUID(); + String remoteStoreRepository = ((SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource()) + .sourceRemoteStoreRepository(); + if (remoteStoreRepository == null) { + remoteStoreRepository = shallowCopyShardMetadata.getRemoteStoreRepository(); + } + + RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService); + RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( + remoteStoreRepository, + indexUUID, + String.valueOf(shardId.id()) + ); + indexShard.syncSegmentsFromGivenRemoteSegmentStore(true, sourceRemoteDirectory, primaryTerm, commitGeneration); + final Store store = indexShard.store(); + if (indexShard.indexSettings.isRemoteTranslogStoreEnabled() == false) { + bootstrap(indexShard, store); + } else { + bootstrapForSnapshot(indexShard, store); + } + assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; + writeEmptyRetentionLeasesFile(indexShard); + indexShard.recoveryState().getIndex().setFileDetailsComplete(); + if (indexShard.indexSettings.isRemoteStoreEnabled()) { + indexShard.openEngineAndSkipTranslogRecoveryFromSnapshot(); + } else { + indexShard.openEngineAndRecoverFromTranslog(); + } + indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm()); + indexShard.finalizeRecovery(); + indexShard.postRecovery("restore done"); + + listener.onResponse(true); + } else { + listener.onResponse(false); + } + } catch (Exception e) { + listener.onFailure(e); + } + } + private boolean canRecover(IndexShard indexShard) { if (indexShard.state() == IndexShardState.CLOSED) { // got closed on us, just ignore this recovery @@ -597,10 +668,18 @@ private void restore( } final ActionListener restoreListener = ActionListener.wrap(v -> { final Store store = indexShard.store(); - bootstrap(indexShard, store); + if (indexShard.indexSettings.isRemoteTranslogStoreEnabled() == false) { + bootstrap(indexShard, store); + } else { + bootstrapForSnapshot(indexShard, store); + } assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; writeEmptyRetentionLeasesFile(indexShard); - indexShard.openEngineAndRecoverFromTranslog(); + if (indexShard.indexSettings.isRemoteStoreEnabled()) { + indexShard.openEngineAndSkipTranslogRecoveryFromSnapshot(); + } else { + indexShard.openEngineAndRecoverFromTranslog(); + } indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm()); indexShard.finalizeRecovery(); indexShard.postRecovery("restore done"); @@ -644,6 +723,21 @@ private void restore( } } + private void bootstrapForSnapshot(final IndexShard indexShard, final Store store) throws IOException { + store.bootstrapNewHistory(); + final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); + final long localCheckpoint = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); + String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY); + Translog.createEmptyTranslog( + indexShard.shardPath().resolveTranslog(), + shardId, + localCheckpoint, + indexShard.getPendingPrimaryTerm(), + translogUUID, + FileChannel::open + ); + } + private void bootstrap(final IndexShard indexShard, final Store store) throws IOException { store.bootstrapNewHistory(); final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 15c6fbea99148..addd8a24af9c5 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -126,6 +126,24 @@ public RemoteSegmentMetadata init() throws IOException { return remoteSegmentMetadata; } + /** + * Initializes the cache to a specific commit which keeps track of all the segment files uploaded to the + * remote segment store. + * this is currently used to restore snapshots, where we want to copy segment files from a given commit. + * TODO: check if we can return read only RemoteSegmentStoreDirectory object from here. + * @throws IOException if there were any failures in reading the metadata file + */ + public RemoteSegmentMetadata initializeToSpecificCommit(long primaryTerm, long commitGeneration) throws IOException { + String metadataFile = getMetadataFileForCommit(primaryTerm, commitGeneration); + RemoteSegmentMetadata remoteSegmentMetadata = readMetadataFile(metadataFile); + if (remoteSegmentMetadata != null) { + this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(remoteSegmentMetadata.getMetadata()); + } else { + this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(); + } + return remoteSegmentMetadata; + } + /** * Read the latest metadata file to get the list of segments uploaded to the remote segment store. * We upload a metadata file per refresh, but it is not unique per refresh. Refresh metadata file is unique for a given commit. @@ -485,6 +503,7 @@ public void uploadMetadata( new RemoteSegmentMetadata( RemoteSegmentMetadata.fromMapOfStrings(uploadedSegments), segmentInfoSnapshotByteArray, + primaryTerm, segmentInfosSnapshot.getGeneration() ) ); diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 388f80ea3e480..03995d5913fb3 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -43,6 +43,11 @@ public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throw String repositoryName = indexSettings.getRemoteStoreRepository(); String indexUUID = indexSettings.getIndex().getUUID(); String shardId = String.valueOf(path.getShardId().getId()); + + return newDirectory(repositoryName, indexUUID, shardId); + } + + public Directory newDirectory(String repositoryName, String indexUUID, String shardId) throws IOException { try (Repository repository = repositoriesService.get().repository(repositoryName)) { assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobPath commonBlobPath = ((BlobStoreRepository) repository).basePath(); diff --git a/server/src/main/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadata.java b/server/src/main/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadata.java index 2a84fbfb89c93..9a479346ff711 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadata.java +++ b/server/src/main/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadata.java @@ -38,16 +38,19 @@ public class RemoteSegmentMetadata { private final byte[] segmentInfosBytes; + private final long primaryTerm; private final long generation; public RemoteSegmentMetadata( Map metadata, byte[] segmentInfosBytes, + long primaryTerm, long generation ) { this.metadata = metadata; this.segmentInfosBytes = segmentInfosBytes; this.generation = generation; + this.primaryTerm = primaryTerm; } /** @@ -66,6 +69,10 @@ public long getGeneration() { return generation; } + public long getPrimaryTerm() { + return primaryTerm; + } + /** * Generate {@code Map} from {@link RemoteSegmentMetadata} * @return {@code Map} @@ -93,6 +100,7 @@ public static Map f public void write(IndexOutput out) throws IOException { out.writeMapOfStrings(toMapOfStrings()); out.writeLong(generation); + out.writeLong(primaryTerm); out.writeLong(segmentInfosBytes.length); out.writeBytes(segmentInfosBytes, segmentInfosBytes.length); } @@ -100,9 +108,10 @@ public void write(IndexOutput out) throws IOException { public static RemoteSegmentMetadata read(IndexInput indexInput) throws IOException { Map metadata = indexInput.readMapOfStrings(); long generation = indexInput.readLong(); + long primaryTerm = indexInput.readLong(); int byteArraySize = (int) indexInput.readLong(); byte[] segmentInfosBytes = new byte[byteArraySize]; indexInput.readBytes(segmentInfosBytes, 0, byteArraySize); - return new RemoteSegmentMetadata(RemoteSegmentMetadata.fromMapOfStrings(metadata), segmentInfosBytes, generation); + return new RemoteSegmentMetadata(RemoteSegmentMetadata.fromMapOfStrings(metadata), segmentInfosBytes, primaryTerm, generation); } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 6ebb1bf7d2252..04057b581e8d9 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -423,6 +423,20 @@ private void deleteStaleRemotePrimaryTermsAndMetadataFiles() { } } + public static void cleanup(Repository repository, ShardId shardId, ThreadPool threadPool) throws IOException { + assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; + BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; + FileTransferTracker fileTransferTracker = new FileTransferTracker(shardId); + TranslogTransferManager translogTransferManager = buildTranslogTransferManager( + blobStoreRepository, + threadPool, + shardId, + fileTransferTracker + ); + // clean up all remote translog files + translogTransferManager.deleteTranslogFiles(); + } + protected void onDelete() { if (primaryModeSupplier.getAsBoolean() == false) { logger.trace("skipped delete translog"); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 58aca00d2e9d3..f6405bc9b5c82 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -361,6 +361,11 @@ public void onFailure(Exception e) { }); } + public void deleteTranslogFiles() throws IOException { + transferService.delete(remoteMetadataTransferPath); + transferService.delete(remoteDataTransferPath); + } + /** * Deletes list of translog files asynchronously using the {@code REMOTE_PURGE} threadpool. * diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java index 4a0fab82f9adc..e4b251914fa0b 100644 --- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java @@ -107,6 +107,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.CLOSED; import static org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED; import static org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.FAILURE; @@ -544,7 +545,19 @@ private void createIndices(final ClusterState state) { AllocatedIndex indexService = null; try { - indexService = indicesService.createIndex(indexMetadata, builtInIndexListener, true); + List updatedIndexEventListeners = new ArrayList<>(builtInIndexListener); + if (entry.getValue().size() > 0 + && entry.getValue().get(0).recoverySource().getType() == Type.SNAPSHOT + && indexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false)) { + final IndexEventListener refreshListenerAfterSnapshotRestore = new IndexEventListener() { + @Override + public void afterIndexShardStarted(IndexShard indexShard) { + indexShard.refresh("refresh to upload metadata to remote store"); + } + }; + updatedIndexEventListeners.add(refreshListenerAfterSnapshotRestore); + } + indexService = indicesService.createIndex(indexMetadata, updatedIndexEventListeners, true); if (indexService.updateMapping(null, indexMetadata) && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh( state.nodes().getClusterManagerNode(), diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index 88e14a4dff3a0..b108e2da1ab04 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -45,6 +45,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.snapshots.SnapshotId; @@ -219,6 +220,15 @@ public void restoreShard( in.restoreShard(store, snapshotId, indexId, snapshotShardId, recoveryState, listener); } + @Override + public RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetadata( + SnapshotId snapshotId, + IndexId indexId, + ShardId snapshotShardId + ) { + return in.getRemoteStoreShallowCopyShardMetadata(snapshotId, indexId, snapshotShardId); + } + @Override public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { return in.getShardSnapshotStatus(snapshotId, indexId, shardId); diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 045b7ad348a76..c08369b79452d 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -46,6 +46,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.snapshots.SnapshotId; @@ -304,6 +305,22 @@ void restoreShard( ActionListener listener ); + /** + * Returns Snapshot Shard Metadata for remote store interop enabled snapshot. + *

+ * The index can be renamed on restore, hence different {@code shardId} and {@code snapshotShardId} are supplied. + * @param snapshotId snapshot id + * @param indexId id of the index in the repository from which the restore is occurring + * @param snapshotShardId shard id (in the snapshot) + */ + default RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetadata( + SnapshotId snapshotId, + IndexId indexId, + ShardId snapshotShardId + ) { + throw new UnsupportedOperationException(); + } + /** * Retrieve shard snapshot status for the stored snapshot * diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index c0d6f49a5ce0d..f04bf83c2f1d1 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -2789,6 +2789,16 @@ public InputStream maybeRateLimitSnapshots(InputStream stream) { return maybeRateLimit(stream, () -> snapshotRateLimiter, snapshotRateLimitingTimeInNanos); } + @Override + public RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetadata( + SnapshotId snapshotId, + IndexId indexId, + ShardId snapshotShardId + ) { + final BlobContainer container = shardContainer(indexId, snapshotShardId); + return loadShallowCopyShardSnapshot(container, snapshotId); + } + @Override public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(shardContainer(indexId, shardId), snapshotId); diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 3d4b04889a5c9..bd162914e830e 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -450,12 +450,25 @@ public ClusterState execute(ClusterState currentState) { final boolean isSearchableSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match( snapshotIndexMetadata.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) ); + final boolean isRemoteStoreShallowCopy = Boolean.TRUE.equals( + snapshotInfo.isRemoteStoreIndexShallowCopyEnabled() + ) && metadata.index(index).getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false); + if (isRemoteStoreShallowCopy && !currentState.getNodes().getMinNodeVersion().onOrAfter(Version.V_3_0_0)) { + throw new SnapshotRestoreException( + snapshot, + "cannot restore shallow copy snapshot for index [" + + index + + "] as some of the nodes in cluster have version less than 2.9" + ); + } final SnapshotRecoverySource recoverySource = new SnapshotRecoverySource( restoreUUID, snapshot, snapshotInfo.version(), repositoryData.resolveIndexId(index), - isSearchableSnapshot + isSearchableSnapshot, + isRemoteStoreShallowCopy, + request.getSourceRemoteStoreRepository() ); final Version minIndexCompatibilityVersion; if (isSearchableSnapshot && isSearchableSnapshotsExtendedCompatibilityEnabled()) { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index 737e7b2e4887b..bb55ac810ed09 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -112,6 +112,10 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { instance.snapshotUuid(randomBoolean() ? null : randomAlphaOfLength(10)); } + if (randomBoolean()) { + instance.setSourceRemoteStoreRepository(randomAlphaOfLengthBetween(5, 10)); + } + return instance; } diff --git a/server/src/test/java/org/opensearch/cluster/routing/RecoverySourceTests.java b/server/src/test/java/org/opensearch/cluster/routing/RecoverySourceTests.java index e4aae52f41e68..a5c006362a20c 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RecoverySourceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RecoverySourceTests.java @@ -36,6 +36,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.repositories.IndexId; +import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -54,6 +56,28 @@ public void testSerialization() throws IOException { assertEquals(recoverySource, serializedRecoverySource); } + public void testSerializationSnapshotRecoverySource() throws IOException { + boolean isSearchableSnapshot = randomBoolean(); + boolean isRemoteStoreShallowCopyEnabled = randomBoolean(); + String sourceRemoteStoreRepo = "test-remote-repo"; + RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource( + UUIDs.randomBase64UUID(), + new Snapshot("repo", new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID())), + Version.CURRENT, + new IndexId("some_index", UUIDs.randomBase64UUID(random())), + isSearchableSnapshot, + isRemoteStoreShallowCopyEnabled, + sourceRemoteStoreRepo + ); + BytesStreamOutput out = new BytesStreamOutput(); + recoverySource.writeTo(out); + RecoverySource serializedRecoverySource = RecoverySource.readFrom(out.bytes().streamInput()); + assertEquals(recoverySource.getType(), serializedRecoverySource.getType()); + assertEquals(recoverySource, serializedRecoverySource); + assertEquals(recoverySource.remoteStoreIndexShallowCopy(), isRemoteStoreShallowCopyEnabled); + assertEquals(recoverySource.isSearchableSnapshot(), isSearchableSnapshot); + } + public void testRecoverySourceTypeOrder() { assertEquals(RecoverySource.Type.EMPTY_STORE.ordinal(), 0); assertEquals(RecoverySource.Type.EXISTING_STORE.ordinal(), 1); diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 1c0ebf17285f7..58527dbea5791 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -49,6 +50,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; import org.junit.Assert; +import org.opensearch.common.io.PathUtils; import org.opensearch.core.Assertions; import org.opensearch.OpenSearchException; import org.opensearch.Version; @@ -127,6 +129,7 @@ import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreStats; import org.opensearch.index.store.StoreUtils; @@ -188,7 +191,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; - +import java.util.Collection; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -2797,6 +2800,78 @@ public void restoreShard( closeShards(target); } + public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { + String remoteStorePath = createTempDir().toString(); + IndexShard source = newStartedShard( + true, + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStorePath + "__test") + .build(), + new InternalEngineFactory() + ); + indexDoc(source, "_doc", "1"); + indexDoc(source, "_doc", "2"); + source.refresh("test"); + assertDocs(source, "1", "2"); + indexDoc(source, "_doc", "3"); + source.refresh("test"); + flushShard(source); + + indexDoc(source, "_doc", "5"); + source.refresh("test"); + + indexDoc(source, "_doc", "4"); + source.refresh("test"); + + long primaryTerm; + long commitGeneration; + try (GatedCloseable segmentInfosGatedCloseable = source.getSegmentInfosSnapshot()) { + SegmentInfos segmentInfos = segmentInfosGatedCloseable.get(); + primaryTerm = source.getOperationPrimaryTerm(); + commitGeneration = segmentInfos.getGeneration(); + } + Collection lastCommitedSegmentsInSource = SegmentInfos.readLatestCommit(source.store().directory()).files(false); + + closeShards(source); + + RemoteSegmentStoreDirectory tempRemoteSegmentDirectory = createRemoteSegmentStoreDirectory( + source.shardId(), + PathUtils.get(remoteStorePath) + ); + + IndexShard target = newStartedShard( + true, + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .build(), + new InternalEngineFactory() + ); + ShardRouting routing = ShardRoutingHelper.initWithSameId( + target.routingEntry(), + RecoverySource.ExistingStoreRecoverySource.INSTANCE + ); + routing = ShardRoutingHelper.newWithRestoreSource(routing, new RecoverySource.EmptyStoreRecoverySource()); + + target = reinitShard(target, routing); + + target.syncSegmentsFromGivenRemoteSegmentStore(false, tempRemoteSegmentDirectory, primaryTerm, commitGeneration); + RemoteSegmentStoreDirectory remoteStoreDirectory = ((RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) target + .remoteStore() + .directory()).getDelegate()).getDelegate()); + Collection uploadFiles = remoteStoreDirectory.getSegmentsUploadedToRemoteStore().keySet(); + assertTrue(uploadFiles.containsAll(lastCommitedSegmentsInSource)); + assertTrue( + "Failed to sync all files to new shard", + List.of(target.store().directory().listAll()).containsAll(lastCommitedSegmentsInSource) + ); + Directory storeDirectory = ((FilterDirectory) ((FilterDirectory) target.store().directory()).getDelegate()).getDelegate(); + ((BaseDirectoryWrapper) storeDirectory).setCheckIndexOnClose(false); + closeShards(target); + } + public void testRefreshLevelRestoreShardFromRemoteStore() throws IOException { testRestoreShardFromRemoteStore(false); } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index fec9b04d6e371..3417e7b0aee04 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -228,7 +228,8 @@ private Map getDummyMetadata(String prefix, int commitGeneration * @return ByteArrayIndexInput: metadata file bytes with header and footer * @throws IOException IOException */ - private ByteArrayIndexInput createMetadataFileBytes(Map segmentFilesMap, long generation) throws IOException { + private ByteArrayIndexInput createMetadataFileBytes(Map segmentFilesMap, long generation, long primaryTerm) + throws IOException { ByteBuffersDataOutput byteBuffersIndexOutput = new ByteBuffersDataOutput(); segmentInfos.write(new ByteBuffersIndexOutput(byteBuffersIndexOutput, "", "")); byte[] byteArray = byteBuffersIndexOutput.toArrayCopy(); @@ -238,6 +239,7 @@ private ByteArrayIndexInput createMetadataFileBytes(Map segmentF CodecUtil.writeHeader(indexOutput, RemoteSegmentMetadata.METADATA_CODEC, RemoteSegmentMetadata.CURRENT_VERSION); indexOutput.writeMapOfStrings(segmentFilesMap); indexOutput.writeLong(generation); + indexOutput.writeLong(primaryTerm); indexOutput.writeLong(byteArray.length); indexOutput.writeBytes(byteArray, byteArray.length); CodecUtil.writeFooter(indexOutput); @@ -261,14 +263,14 @@ private Map> populateMetadata() throws IOException { ); when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc"), 1) + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc"), 1, 5) ); when(remoteMetadataDirectory.openInput("metadata__1__6__pqr", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__6__pqr"), 1) + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__6__pqr"), 1, 6) ); when(remoteMetadataDirectory.openInput("metadata__2__1__zxv", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv"), 1), - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv"), 1) + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv"), 1, 2), + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv"), 1, 2) ); return metadataFilenameContentMapping; @@ -503,7 +505,7 @@ public void testGetSegmentsUploadedToRemoteStore() throws IOException { ); when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc"), 1) + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc"), 1, 5) ); assert (remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore(testPrimaryTerm, testGeneration).containsKey("segments_5")); @@ -577,7 +579,9 @@ public void testContainsFile() throws IOException { metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512"); metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024"); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(createMetadataFileBytes(metadata, 1)); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( + createMetadataFileBytes(metadata, 1, 5) + ); remoteSegmentStoreDirectory.init(); diff --git a/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java b/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java index 3bf7781fb909f..cc0764a6700b1 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java @@ -61,6 +61,7 @@ public void testReadContentNoSegmentInfos() throws IOException { Map expectedOutput = getDummyData(); indexOutput.writeMapOfStrings(expectedOutput); indexOutput.writeLong(1234); + indexOutput.writeLong(1234); indexOutput.writeLong(0); indexOutput.writeBytes(new byte[0], 0); indexOutput.close(); @@ -77,6 +78,7 @@ public void testReadContentWithSegmentInfos() throws IOException { Map expectedOutput = getDummyData(); indexOutput.writeMapOfStrings(expectedOutput); indexOutput.writeLong(1234); + indexOutput.writeLong(1234); ByteBuffersIndexOutput segmentInfosOutput = new ByteBuffersIndexOutput(new ByteBuffersDataOutput(), "test", "resource"); segmentInfos.write(segmentInfosOutput); byte[] segmentInfosBytes = segmentInfosOutput.toArrayCopy(); @@ -103,6 +105,7 @@ public void testWriteContent() throws IOException { RemoteSegmentMetadata remoteSegmentMetadata = new RemoteSegmentMetadata( RemoteSegmentMetadata.fromMapOfStrings(expectedOutput), segmentInfosBytes, + 1234, 1234 ); remoteSegmentMetadataHandler.writeContent(indexOutput, remoteSegmentMetadata); @@ -113,6 +116,7 @@ public void testWriteContent() throws IOException { ); assertEquals(expectedOutput, metadata.toMapOfStrings()); assertEquals(1234, metadata.getGeneration()); + assertEquals(1234, metadata.getPrimaryTerm()); assertArrayEquals(segmentInfosBytes, metadata.getSegmentInfosBytes()); } diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index 43d371bf5a187..f5295bead19a4 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -58,6 +58,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; @@ -342,6 +343,15 @@ public void restoreShard( } + @Override + public RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetadata( + SnapshotId snapshotId, + IndexId indexId, + ShardId snapshotShardId + ) { + return null; + } + @Override public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { return null; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 8e0ee6b16ed48..105ccef500ce8 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -50,7 +50,12 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.index.IndexModule; +import org.opensearch.index.IndexService; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.ShardId; +import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.RemoteBufferedOutputDirectory; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; @@ -117,11 +122,7 @@ protected void assertSnapshotOrGenericThread() { @Override protected Settings nodeSettings() { - return Settings.builder() - .put(super.nodeSettings()) - .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") - .put(FeatureFlags.REMOTE_STORE, "true") - .build(); + return Settings.builder().put(super.nodeSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); } public void testRetrieveSnapshots() throws Exception { @@ -326,12 +327,89 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); - List snapshotIds = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository) - .getSnapshotIds() + RepositoryData repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); + IndexId indexId = repositoryData.resolveIndexId(remoteStoreIndexName); + + List snapshotIds = repositoryData.getSnapshotIds() .stream() .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())) .collect(Collectors.toList()); assertThat(snapshotIds, equalTo(originalSnapshots)); + + // shallow copy shard metadata - getRemoteStoreShallowCopyShardMetadata + RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = repository.getRemoteStoreShallowCopyShardMetadata( + snapshotId2, + indexId, + new ShardId(remoteStoreIndexName, indexId.getId(), 0) + ); + assertEquals(shardShallowCopySnapshot.getRemoteStoreRepository(), remoteStoreRepositoryName); + } + + public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + final Client client = client(); + final String snapshotRepositoryName = "test-repo"; + final String remoteStoreRepositoryName = "test-rs-repo"; + + logger.info("--> creating snapshot repository"); + + Settings snapshotRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, snapshotRepositoryName, snapshotRepoSettings); + + logger.info("--> creating remote store repository"); + Settings remoteStoreRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); + + logger.info("--> creating a remote store enabled index and indexing documents"); + final String remoteStoreIndexName = "test-rs-idx"; + Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + createIndex(remoteStoreIndexName, indexSettings); + indexDocuments(client, remoteStoreIndexName); + + logger.info("--> create remote index shallow snapshot"); + Settings snapshotRepoSettingsForShallowCopy = Settings.builder() + .put(snapshotRepoSettings) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE) + .build(); + updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); + + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") + .setWaitForCompletion(true) + .setIndices(remoteStoreIndexName) + .get(); + final SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); + assert lockFiles[0].endsWith(snapshotId.getUUID() + ".lock"); + + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); + RepositoryData repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); + IndexSettings indexSetting = getIndexSettings(remoteStoreIndexName); + IndexId indexId = repositoryData.resolveIndexId(remoteStoreIndexName); + RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = repository.getRemoteStoreShallowCopyShardMetadata( + snapshotId, + indexId, + new ShardId(remoteStoreIndexName, indexSetting.getUUID(), 0) + ); + assertEquals(shardShallowCopySnapshot.getRemoteStoreRepository(), remoteStoreRepositoryName); + assertEquals(shardShallowCopySnapshot.getIndexUUID(), indexSetting.getUUID()); + assertEquals(shardShallowCopySnapshot.getRepositoryBasePath(), ""); + } + + private IndexSettings getIndexSettings(String indexName) { + final IndicesService indicesService = getInstanceFromNode(IndicesService.class); + final IndexService indexService = indicesService.indexService(resolveIndex(indexName)); + return indexService.getIndexSettings(); } // Validate Scenario remoteStoreShallowCopy Snapshot -> remoteStoreShallowCopy Snapshot diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 659f473403ec8..ea9e9342673db 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -67,6 +67,7 @@ import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -101,6 +102,9 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; +import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; +import org.opensearch.index.store.RemoteBufferedOutputDirectory; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.Translog; @@ -574,7 +578,14 @@ protected IndexShard newShard( RemoteRefreshSegmentPressureService remoteRefreshSegmentPressureService = null; if (indexSettings.isRemoteStoreEnabled()) { if (remoteStore == null) { - remoteStore = createRemoteStore(createTempDir(), routing, indexMetadata); + Path remoteStorePath; + String remoteStoreRepository = indexSettings.getRemoteStoreRepository(); + if (remoteStoreRepository != null && remoteStoreRepository.endsWith("__test")) { + remoteStorePath = PathUtils.get(remoteStoreRepository.replace("__test", "")); + } else { + remoteStorePath = createTempDir(); + } + remoteStore = createRemoteStore(remoteStorePath, routing, indexMetadata); } remoteRefreshSegmentPressureService = new RemoteRefreshSegmentPressureService(clusterService, indexSettings.getSettings()); } @@ -642,21 +653,30 @@ protected RepositoriesService createRepositoriesService() { protected Store createRemoteStore(Path path, ShardRouting shardRouting, IndexMetadata metadata) throws IOException { Settings nodeSettings = Settings.builder().put("node.name", shardRouting.currentNodeId()).build(); + ShardId shardId = shardRouting.shardId(); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = createRemoteSegmentStoreDirectory(shardId, path); + return createStore(shardId, new IndexSettings(metadata, nodeSettings), remoteSegmentStoreDirectory); + } - ShardId shardId = new ShardId("index", "_na_", 0); + protected RemoteSegmentStoreDirectory createRemoteSegmentStoreDirectory(ShardId shardId, Path path) throws IOException { NodeEnvironment.NodePath remoteNodePath = new NodeEnvironment.NodePath(path); ShardPath remoteShardPath = new ShardPath(false, remoteNodePath.resolve(shardId), remoteNodePath.resolve(shardId), shardId); RemoteDirectory dataDirectory = newRemoteDirectory(remoteShardPath.resolveIndex()); RemoteDirectory metadataDirectory = newRemoteDirectory(remoteShardPath.resolveIndex()); - RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, null); - return createStore(shardId, new IndexSettings(metadata, nodeSettings), remoteSegmentStoreDirectory); + RemoteStoreLockManager remoteStoreLockManager = new RemoteStoreMetadataLockManager( + new RemoteBufferedOutputDirectory(getBlobContainer(remoteShardPath.resolveIndex())) + ); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager); } private RemoteDirectory newRemoteDirectory(Path f) throws IOException { + return new RemoteDirectory(getBlobContainer(f)); + } + + protected BlobContainer getBlobContainer(Path f) throws IOException { FsBlobStore fsBlobStore = new FsBlobStore(1024, f, false); BlobPath blobPath = new BlobPath(); - BlobContainer fsBlobContainer = new FsBlobContainer(fsBlobStore, blobPath, f); - return new RemoteDirectory(fsBlobContainer); + return new FsBlobContainer(fsBlobStore, blobPath, f); } /** diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java index 28660ba834a65..ad515f2405f1d 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java @@ -240,15 +240,18 @@ private static void assertSnapshotUUIDs(BlobStoreRepository repository, Reposito final BlobContainer repoRoot = repository.blobContainer(); final Collection snapshotIds = repositoryData.getSnapshotIds(); final List expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).collect(Collectors.toList()); - for (String prefix : new String[] { BlobStoreRepository.SNAPSHOT_PREFIX, BlobStoreRepository.METADATA_PREFIX }) { - final Collection foundSnapshotUUIDs = repoRoot.listBlobs() - .keySet() - .stream() - .filter(p -> p.startsWith(prefix)) - .map(p -> p.replace(prefix, "").replace(".dat", "")) - .collect(Collectors.toSet()); - assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY))); + Collection foundSnapshotUUIDs = new HashSet<>(); + for (String prefix : new String[] { BlobStoreRepository.SNAPSHOT_PREFIX, BlobStoreRepository.SHALLOW_SNAPSHOT_PREFIX }) { + foundSnapshotUUIDs.addAll( + repoRoot.listBlobs() + .keySet() + .stream() + .filter(p -> p.startsWith(prefix)) + .map(p -> p.replace(prefix, "").replace(".dat", "")) + .collect(Collectors.toSet()) + ); } + assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY))); final BlobContainer indicesContainer = repository.getBlobContainer().children().get("indices"); final Map indices; @@ -303,10 +306,16 @@ private static void assertSnapshotUUIDs(BlobStoreRepository repository, Reposito .stream() .noneMatch(shardFailure -> shardFailure.index().equals(index) && shardFailure.shardId() == shardId)) { final Map shardPathContents = shardContainer.listBlobs(); - assertThat( - shardPathContents, - hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())) + + assertTrue( + shardPathContents.containsKey( + String.format(Locale.ROOT, BlobStoreRepository.SHALLOW_SNAPSHOT_NAME_FORMAT, snapshotId.getUUID()) + ) + || shardPathContents.containsKey( + String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID()) + ) ); + assertThat( shardPathContents.keySet() .stream() diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index 9933297aa1c96..ddf9f3e96b9b4 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -105,6 +105,7 @@ public abstract class AbstractSnapshotIntegTestCase extends OpenSearchIntegTestCase { + protected final static String TEST_REMOTE_STORE_REPO_SUFFIX = "__rs"; private static final String OLD_VERSION_SNAPSHOT_PREFIX = "old-version-snapshot-"; // Large snapshot pool settings to set up nodes for tests involving multiple repositories that need to have enough @@ -148,14 +149,19 @@ public void verifyNoLeakedListeners() throws Exception { @After public void assertRepoConsistency() { if (skipRepoConsistencyCheckReason == null) { - clusterAdmin().prepareGetRepositories().get().repositories().forEach(repositoryMetadata -> { - final String name = repositoryMetadata.name(); - if (repositoryMetadata.settings().getAsBoolean("readonly", false) == false) { - clusterAdmin().prepareDeleteSnapshot(name, OLD_VERSION_SNAPSHOT_PREFIX + "*").get(); - clusterAdmin().prepareCleanupRepository(name).get(); - } - BlobStoreTestUtil.assertRepoConsistency(internalCluster(), name); - }); + clusterAdmin().prepareGetRepositories() + .get() + .repositories() + .stream() + .filter(repositoryMetadata -> !repositoryMetadata.name().endsWith(TEST_REMOTE_STORE_REPO_SUFFIX)) + .forEach(repositoryMetadata -> { + final String name = repositoryMetadata.name(); + if (repositoryMetadata.settings().getAsBoolean("readonly", false) == false) { + clusterAdmin().prepareDeleteSnapshot(name, OLD_VERSION_SNAPSHOT_PREFIX + "*").get(); + clusterAdmin().prepareCleanupRepository(name).get(); + } + BlobStoreTestUtil.assertRepoConsistency(internalCluster(), name); + }); } else { logger.info("--> skipped repo consistency checks because [{}]", skipRepoConsistencyCheckReason); } @@ -367,10 +373,36 @@ protected void createRepository(String repoName, String type, Settings.Builder s assertAcked(clusterAdmin().preparePutRepository(repoName).setType(type).setSettings(settings)); } + protected void updateRepository(String repoName, String type, Settings.Builder settings) { + logger.info("--> updating repository [{}] [{}]", repoName, type); + assertAcked(clusterAdmin().preparePutRepository(repoName).setType(type).setSettings(settings)); + } + protected void createRepository(String repoName, String type, Path location) { createRepository(repoName, type, Settings.builder().put("location", location)); } + protected Settings.Builder getRepositorySettings(Path location, boolean shallowCopyEnabled) { + Settings.Builder settingsBuilder = randomRepositorySettings(); + settingsBuilder.put("location", location); + if (shallowCopyEnabled) { + settingsBuilder.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true); + } + return settingsBuilder; + } + + protected Settings.Builder getRepositorySettings(Path location, String basePath, boolean shallowCopyEnabled) { + Settings.Builder settingsBuilder = randomRepositorySettings(); + settingsBuilder.put("location", location); + if (shallowCopyEnabled) { + settingsBuilder.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true); + } + if (basePath != null) { + settingsBuilder.put("base_path", basePath); + } + return settingsBuilder; + } + protected void createRepository(String repoName, String type) { createRepository(repoName, type, randomRepositorySettings()); } From 3947cb48faf1522fb0e90d234bc228161be11714 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Tue, 27 Jun 2023 16:28:14 -0700 Subject: [PATCH 012/186] Remove minimum file cache size restriction (#8294) Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 1 + .../remote/filecache/FileCacheFactory.java | 18 +----- ...TransportClearIndicesCacheActionTests.java | 2 +- .../filecache/FileCacheCleanerTests.java | 2 +- .../remote/filecache/FileCacheTests.java | 55 ++++++++----------- 5 files changed, 29 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50e1fe78daf5d..0ffb14d6750c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Removed - Remove `COMPRESSOR` variable from `CompressorFactory` and use `DEFLATE_COMPRESSOR` instead ([7907](https://github.com/opensearch-project/OpenSearch/pull/7907)) +- Remove concurrency based minimum file cache size restriction ([#8294](https://github.com/opensearch-project/OpenSearch/pull/8294)) ### Fixed - Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924 diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java index 5ed1f5e0ee21b..f23e057196096 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java @@ -11,7 +11,6 @@ import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.cache.RemovalReason; import org.opensearch.index.store.remote.utils.cache.SegmentedCache; -import org.opensearch.index.store.remote.file.OnDemandBlockSnapshotIndexInput; import java.nio.file.Files; import java.nio.file.Path; @@ -39,24 +38,11 @@ public class FileCacheFactory { public static FileCache createConcurrentLRUFileCache(long capacity, CircuitBreaker circuitBreaker) { - return createFileCache(createDefaultBuilder().capacity(capacity).build(), circuitBreaker); + return new FileCache(createDefaultBuilder().capacity(capacity).build(), circuitBreaker); } public static FileCache createConcurrentLRUFileCache(long capacity, int concurrencyLevel, CircuitBreaker circuitBreaker) { - return createFileCache(createDefaultBuilder().capacity(capacity).concurrencyLevel(concurrencyLevel).build(), circuitBreaker); - } - - private static FileCache createFileCache(SegmentedCache segmentedCache, CircuitBreaker circuitBreaker) { - /* - * Since OnDemandBlockSnapshotIndexInput.Builder.DEFAULT_BLOCK_SIZE is not overridden then it will be upper bound for max IndexInput - * size on disk. A single IndexInput size should always be more than a single segment in segmented cache. A FileCache capacity might - * be defined with large capacity (> IndexInput block size) but due to segmentation and concurrency factor, that capacity is - * distributed equally across segments. - */ - if (segmentedCache.getPerSegmentCapacity() <= OnDemandBlockSnapshotIndexInput.Builder.DEFAULT_BLOCK_SIZE) { - throw new IllegalStateException("FileSystem Cache per segment capacity is less than single IndexInput default block size"); - } - return new FileCache(segmentedCache, circuitBreaker); + return new FileCache(createDefaultBuilder().capacity(capacity).concurrencyLevel(concurrencyLevel).build(), circuitBreaker); } private static SegmentedCache.Builder createDefaultBuilder() { diff --git a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java index 85aa60d6f308b..57a41f08f73f2 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java @@ -85,7 +85,7 @@ public void testOnShardOperation() throws IOException { when(shardRouting.shardId()).thenReturn(shardId); final ShardPath shardPath = ShardPath.loadFileCachePath(nodeEnvironment, shardId); final Path cacheEntryPath = shardPath.getDataPath(); - final FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(1024 * 1024 * 1024, 16, new NoopCircuitBreaker("")); + final FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(1024 * 1024, 16, new NoopCircuitBreaker("")); when(testNode.fileCache()).thenReturn(fileCache); when(testNode.getNodeEnvironment()).thenReturn(nodeEnvironment); diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java index 3924cd4ed1913..5bdba0513af23 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java @@ -48,7 +48,7 @@ public class FileCacheCleanerTests extends OpenSearchTestCase { ); private final FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache( - 1024 * 1024 * 1024, + 1024 * 1024, 1, new NoopCircuitBreaker(CircuitBreaker.REQUEST) ); diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java index 43a5c04b59f83..02b6a48b6f48e 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java @@ -31,7 +31,6 @@ public class FileCacheTests extends OpenSearchTestCase { // but fatal to these tests private final static int CONCURRENCY_LEVEL = 16; private final static int MEGA_BYTES = 1024 * 1024; - private final static int GIGA_BYTES = 1024 * 1024 * 1024; private final static String FAKE_PATH_SUFFIX = "Suffix"; private Path path; @@ -66,15 +65,9 @@ private void createFile(String indexName, String shardId, String fileName) throw Files.write(filePath, "test-data".getBytes()); } - public void testCreateCacheWithSmallSegments() { - assertThrows(IllegalStateException.class, () -> { - FileCacheFactory.createConcurrentLRUFileCache(1000, CONCURRENCY_LEVEL, new NoopCircuitBreaker(CircuitBreaker.REQUEST)); - }); - } - // test get method public void testGet() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(8 * MEGA_BYTES); for (int i = 0; i < 4; i++) { fileCache.put(createPath(Integer.toString(i)), new StubCachedIndexInput(8 * MEGA_BYTES)); } @@ -86,27 +79,27 @@ public void testGet() { public void testGetThrowException() { assertThrows(NullPointerException.class, () -> { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); fileCache.get(null); }); } public void testPutThrowException() { assertThrows(NullPointerException.class, () -> { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); fileCache.put(null, null); }); } public void testPutThrowCircuitBreakingException() { - FileCache fileCache = createCircuitBreakingFileCache(GIGA_BYTES); + FileCache fileCache = createCircuitBreakingFileCache(MEGA_BYTES); Path path = createPath("0"); assertThrows(CircuitBreakingException.class, () -> fileCache.put(path, new StubCachedIndexInput(8 * MEGA_BYTES))); assertNull(fileCache.get(path)); } public void testCompute() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); Path path = createPath("0"); fileCache.put(path, new StubCachedIndexInput(8 * MEGA_BYTES)); fileCache.incRef(path); @@ -117,20 +110,20 @@ public void testCompute() { public void testComputeThrowException() { assertThrows(NullPointerException.class, () -> { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); fileCache.compute(null, null); }); } public void testComputeThrowCircuitBreakingException() { - FileCache fileCache = createCircuitBreakingFileCache(GIGA_BYTES); + FileCache fileCache = createCircuitBreakingFileCache(MEGA_BYTES); Path path = createPath("0"); assertThrows(CircuitBreakingException.class, () -> fileCache.compute(path, (p, i) -> new StubCachedIndexInput(8 * MEGA_BYTES))); assertNull(fileCache.get(path)); } public void testRemove() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { fileCache.put(createPath(Integer.toString(i)), new StubCachedIndexInput(8 * MEGA_BYTES)); } @@ -145,13 +138,13 @@ public void testRemove() { public void testRemoveThrowException() { assertThrows(NullPointerException.class, () -> { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); fileCache.remove(null); }); } public void testIncDecRef() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { fileCache.put(createPath(Integer.toString(i)), new StubCachedIndexInput(8 * MEGA_BYTES)); } @@ -184,7 +177,7 @@ public void testIncDecRef() { public void testIncRefThrowException() { assertThrows(NullPointerException.class, () -> { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); fileCache.incRef(null); }); @@ -192,19 +185,19 @@ public void testIncRefThrowException() { public void testDecRefThrowException() { assertThrows(NullPointerException.class, () -> { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); fileCache.decRef(null); }); } public void testCapacity() { - FileCache fileCache = createFileCache(GIGA_BYTES); - assertEquals(fileCache.capacity(), GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); + assertEquals(fileCache.capacity(), MEGA_BYTES); } public void testSize() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { fileCache.put(createPath(Integer.toString(i)), new StubCachedIndexInput(8 * MEGA_BYTES)); } @@ -213,34 +206,34 @@ public void testSize() { } public void testPrune() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { putAndDecRef(fileCache, i, 8 * MEGA_BYTES); } // before prune - assertEquals(fileCache.size(), 4); + assertTrue(fileCache.size() >= 1); fileCache.prune(); // after prune - assertEquals(fileCache.size(), 0); + assertEquals(0, fileCache.size()); } public void testPruneWithPredicate() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { putAndDecRef(fileCache, i, 8 * MEGA_BYTES); } // before prune - assertEquals(fileCache.size(), 4); + assertTrue(fileCache.size() >= 1); // after prune with false predicate fileCache.prune(path -> false); - assertEquals(fileCache.size(), 4); + assertTrue(fileCache.size() >= 1); // after prune with true predicate fileCache.prune(path -> true); - assertEquals(fileCache.size(), 0); + assertEquals(0, fileCache.size()); } public void testUsage() { @@ -258,7 +251,7 @@ public void testUsage() { } public void testStats() { - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { fileCache.put(createPath(Integer.toString(i)), new StubCachedIndexInput(8 * MEGA_BYTES)); } @@ -284,7 +277,7 @@ public void testCacheRestore() throws IOException { String indexName = "test-index"; String shardId = "0"; createFile(indexName, shardId, "test.0"); - FileCache fileCache = createFileCache(GIGA_BYTES); + FileCache fileCache = createFileCache(MEGA_BYTES); assertEquals(0, fileCache.usage().usage()); Path fileCachePath = path.resolve(NodeEnvironment.CACHE_FOLDER).resolve(indexName).resolve(shardId); fileCache.restoreFromDirectory(List.of(fileCachePath)); From 4c02dd185c3033c6fabf0c52b8b39a019d324ca4 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Tue, 27 Jun 2023 16:33:39 -0700 Subject: [PATCH 013/186] Removed redundant ubuntu precommit check (#8207) * Run precommit before gradle check Signed-off-by: Owais Kazi * Removed redundant precommit for ubuntu Signed-off-by: Owais Kazi * Addressed PR comment Signed-off-by: Owais Kazi --------- Signed-off-by: Owais Kazi --- .github/workflows/gradle-check.yml | 88 +++++++++++++++--------------- .github/workflows/precommit.yml | 4 +- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index ba1dc3dc3b5e6..f895dfc2c1f4d 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -22,34 +22,34 @@ jobs: timeout-minutes: 130 steps: - name: Checkout OpenSearch repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Setup environment variables (PR) if: github.event_name == 'pull_request_target' run: | - echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - name: Setup environment variables (Push) if: github.event_name == 'push' run: | - repo_url="https://github.com/opensearch-project/OpenSearch" - ref_id=$(git rev-parse HEAD) - branch_name=$(git rev-parse --abbrev-ref HEAD) - echo "branch_name=$branch_name" >> $GITHUB_ENV - echo "pr_from_sha=$ref_id" >> $GITHUB_ENV - echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV - echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV - echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV - echo "pr_number=Null" >> $GITHUB_ENV + repo_url="https://github.com/opensearch-project/OpenSearch" + ref_id=$(git rev-parse HEAD) + branch_name=$(git rev-parse --abbrev-ref HEAD) + echo "branch_name=$branch_name" >> $GITHUB_ENV + echo "pr_from_sha=$ref_id" >> $GITHUB_ENV + echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV + echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV + echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV + echo "pr_number=Null" >> $GITHUB_ENV - name: Checkout opensearch-build repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: opensearch-project/opensearch-build ref: main @@ -57,17 +57,17 @@ jobs: - name: Trigger jenkins workflow to run gradle check run: | - set -e - set -o pipefail - bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log + set -e + set -o pipefail + bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log - name: Setup Result Status if: always() run: | - WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'` - RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'` - echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV - echo "result=$RESULT" >> $GITHUB_ENV + WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'` + RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'` + echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV + echo "result=$RESULT" >> $GITHUB_ENV - name: Upload Coverage Report if: success() @@ -81,25 +81,25 @@ jobs: with: issue-number: ${{ env.pr_number }} body: | - ### Gradle Check (Jenkins) Run Completed with: - * **RESULT:** ${{ env.result }} :white_check_mark: - * **URL:** ${{ env.workflow_url }} - * **CommitID:** ${{ env.pr_from_sha }} + ### Gradle Check (Jenkins) Run Completed with: + * **RESULT:** ${{ env.result }} :white_check_mark: + * **URL:** ${{ env.workflow_url }} + * **CommitID:** ${{ env.pr_from_sha }} - name: Extract Test Failure if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }} run: | - TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." + .name)' | uniq -c | sort -n -r | head -n 10` - if [[ "$TEST_FAILURES" != "" ]] - then - echo "test_failures<> $GITHUB_ENV - echo "" >> $GITHUB_ENV - echo "* **TEST FAILURES:**" >> $GITHUB_ENV - echo '```' >> $GITHUB_ENV - echo "$TEST_FAILURES" >> $GITHUB_ENV - echo '```' >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - fi + TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." + .name)' | uniq -c | sort -n -r | head -n 10` + if [[ "$TEST_FAILURES" != "" ]] + then + echo "test_failures<> $GITHUB_ENV + echo "" >> $GITHUB_ENV + echo "* **TEST FAILURES:**" >> $GITHUB_ENV + echo '```' >> $GITHUB_ENV + echo "$TEST_FAILURES" >> $GITHUB_ENV + echo '```' >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + fi - name: Create Comment Flaky if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }} @@ -119,12 +119,12 @@ jobs: with: issue-number: ${{ env.pr_number }} body: | - ### Gradle Check (Jenkins) Run Completed with: - * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }} - * **URL:** ${{ env.workflow_url }} - * **CommitID:** ${{ env.pr_from_sha }} - Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. - Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change? + ### Gradle Check (Jenkins) Run Completed with: + * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }} + * **URL:** ${{ env.workflow_url }} + * **CommitID:** ${{ env.pr_from_sha }} + Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. + Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change? - name: Create Issue On Push Failure if: ${{ github.event_name == 'push' && failure() }} diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index e264d65cdf191..8bbba657737c8 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -1,12 +1,12 @@ name: Gradle Precommit on: [pull_request] - + jobs: precommit: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [windows-latest, macos-latest] # precommit on ubuntu-latest is run as a part of the gradle-check workflow steps: - uses: actions/checkout@v2 - name: Set up JDK 11 From 03bc19220a711853660652a4761c05285f597884 Mon Sep 17 00:00:00 2001 From: suraj kumar Date: Wed, 28 Jun 2023 12:13:49 +0530 Subject: [PATCH 014/186] Add request tracing framework (#7648) * Add request tracing framework Signed-off-by: suranjay * Introduce ThreadContextStatePropagator to propagate request and transient headers within ThreadContext Signed-off-by: Andriy Redko --------- Signed-off-by: suranjay Signed-off-by: Andriy Redko Co-authored-by: Andriy Redko --- CHANGELOG.md | 2 + buildSrc/version.properties | 4 + libs/telemetry/build.gradle | 23 ++ .../org/opensearch/telemetry/Telemetry.java | 31 +++ .../telemetry/metrics/MetricsTelemetry.java | 16 ++ .../telemetry/metrics/package-info.java | 12 ++ .../opensearch/telemetry/package-info.java | 12 ++ .../telemetry/tracing/AbstractSpan.java | 45 ++++ .../telemetry/tracing/DefaultTracer.java | 109 ++++++++++ .../opensearch/telemetry/tracing/Scope.java | 26 +++ .../telemetry/tracing/ScopeImpl.java | 33 +++ .../opensearch/telemetry/tracing/Span.java | 84 ++++++++ .../telemetry/tracing/SpanReference.java | 41 ++++ .../opensearch/telemetry/tracing/Tracer.java | 67 ++++++ .../tracing/TracerContextStorage.java | 35 +++ .../tracing/TracingContextPropagator.java | 34 +++ .../telemetry/tracing/TracingTelemetry.java | 37 ++++ .../telemetry/tracing/noop/NoopTracer.java | 76 +++++++ .../telemetry/tracing/noop/package-info.java | 12 ++ .../telemetry/tracing/package-info.java | 12 ++ .../telemetry/tracing/DefaultTracerTests.java | 122 +++++++++++ plugins/telemetry-otel/build.gradle | 56 +++++ .../opentelemetry-api-1.26.0.jar.sha1 | 1 + .../licenses/opentelemetry-api-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/opentelemetry-api-NOTICE.txt | 0 ...entelemetry-api-logs-1.26.0-alpha.jar.sha1 | 1 + .../opentelemetry-api-logs-LICENSE.txt | 202 ++++++++++++++++++ .../opentelemetry-api-logs-NOTICE.txt | 0 .../opentelemetry-context-1.26.0.jar.sha1 | 1 + .../opentelemetry-context-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/opentelemetry-context-NOTICE.txt | 0 ...telemetry-exporter-logging-1.26.0.jar.sha1 | 1 + ...opentelemetry-exporter-logging-LICENSE.txt | 202 ++++++++++++++++++ .../opentelemetry-exporter-logging-NOTICE.txt | 0 .../opentelemetry-sdk-1.26.0.jar.sha1 | 1 + .../licenses/opentelemetry-sdk-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/opentelemetry-sdk-NOTICE.txt | 0 .../opentelemetry-sdk-common-1.26.0.jar.sha1 | 1 + .../opentelemetry-sdk-common-LICENSE.txt | 202 ++++++++++++++++++ .../opentelemetry-sdk-common-NOTICE.txt | 0 ...entelemetry-sdk-logs-1.26.0-alpha.jar.sha1 | 1 + .../opentelemetry-sdk-logs-LICENSE.txt | 202 ++++++++++++++++++ .../opentelemetry-sdk-logs-NOTICE.txt | 0 .../opentelemetry-sdk-metrics-1.26.0.jar.sha1 | 1 + .../opentelemetry-sdk-metrics-LICENSE.txt | 202 ++++++++++++++++++ .../opentelemetry-sdk-metrics-NOTICE.txt | 0 .../opentelemetry-sdk-trace-1.26.0.jar.sha1 | 1 + .../opentelemetry-sdk-trace-LICENSE.txt | 202 ++++++++++++++++++ .../opentelemetry-sdk-trace-NOTICE.txt | 0 ...pentelemetry-semconv-1.26.0-alpha.jar.sha1 | 1 + .../opentelemetry-semconv-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/opentelemetry-semconv-NOTICE.txt | 0 .../telemetry/OTelTelemetryPlugin.java | 92 ++++++++ .../opensearch/telemetry/package-info.java | 12 ++ .../telemetry/tracing/OTelPropagatedSpan.java | 23 ++ .../tracing/OTelResourceProvider.java | 78 +++++++ .../telemetry/tracing/OTelSpan.java | 70 ++++++ .../telemetry/tracing/OTelTelemetry.java | 41 ++++ .../tracing/OTelTracingContextPropagator.java | 75 +++++++ .../tracing/OTelTracingTelemetry.java | 68 ++++++ .../telemetry/tracing/package-info.java | 12 ++ .../plugin-metadata/plugin-security.policy | 12 ++ .../telemetry/OTelTelemetryPluginTests.java | 50 +++++ .../telemetry/tracing/OTelSpanTests.java | 92 ++++++++ .../OTelTracingContextPropagatorTests.java | 54 +++++ .../tracing/OTelTracingTelemetryTests.java | 68 ++++++ server/build.gradle | 2 + .../common/settings/ClusterSettings.java | 5 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/util/FeatureFlags.java | 7 + .../common/util/concurrent/ThreadContext.java | 74 +++++-- .../ThreadContextStatePropagator.java | 31 +++ .../main/java/org/opensearch/node/Node.java | 21 ++ .../opensearch/plugins/TelemetryPlugin.java | 25 +++ .../TaskThreadContextStatePropagator.java | 38 ++++ .../opensearch/telemetry/TelemetryModule.java | 47 ++++ .../telemetry/TelemetrySettings.java | 42 ++++ .../opensearch/telemetry/package-info.java | 12 ++ .../telemetry/tracing/NoopTracerFactory.java | 33 +++ ...hreadContextBasedTracerContextStorage.java | 93 ++++++++ .../telemetry/tracing/TracerFactory.java | 70 ++++++ .../telemetry/tracing/package-info.java | 12 ++ .../telemetry/TelemetryModuleTests.java | 65 ++++++ .../telemetry/tracing/TracerFactoryTests.java | 68 ++++++ 84 files changed, 4299 insertions(+), 15 deletions(-) create mode 100644 libs/telemetry/build.gradle create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/package-info.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/package-info.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/package-info.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/package-info.java create mode 100644 libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java create mode 100644 plugins/telemetry-otel/build.gradle create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-logs-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-NOTICE.txt create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/package-info.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelPropagatedSpan.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagator.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/package-info.java create mode 100644 plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanTests.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagatorTests.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java create mode 100644 server/src/main/java/org/opensearch/common/util/concurrent/ThreadContextStatePropagator.java create mode 100644 server/src/main/java/org/opensearch/plugins/TelemetryPlugin.java create mode 100644 server/src/main/java/org/opensearch/tasks/TaskThreadContextStatePropagator.java create mode 100644 server/src/main/java/org/opensearch/telemetry/TelemetryModule.java create mode 100644 server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java create mode 100644 server/src/main/java/org/opensearch/telemetry/package-info.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/package-info.java create mode 100644 server/src/test/java/org/opensearch/telemetry/TelemetryModuleTests.java create mode 100644 server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ffb14d6750c5..8c370dbe7b0d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291)) - OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344) - Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928))) +- Add Opentelemetry dependencies ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -83,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020)) - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038)) - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) +- Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index dd64569259c2d..735a9fe4f8255 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -66,3 +66,7 @@ zstd = 1.5.5-3 jzlib = 1.1.3 resteasy = 6.2.4.Final + +# opentelemetry dependencies +opentelemetry = 1.26.0 + diff --git a/libs/telemetry/build.gradle b/libs/telemetry/build.gradle new file mode 100644 index 0000000000000..ce94698836b4f --- /dev/null +++ b/libs/telemetry/build.gradle @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +dependencies { + testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + testImplementation "junit:junit:${versions.junit}" + testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" + testImplementation(project(":test:framework")) { + exclude group: 'org.opensearch', module: 'opensearch-telemetry' + } +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java new file mode 100644 index 0000000000000..6f50699528b6b --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import org.opensearch.telemetry.metrics.MetricsTelemetry; +import org.opensearch.telemetry.tracing.TracingTelemetry; + +/** + * Interface defining telemetry + */ +public interface Telemetry { + + /** + * Provides tracing telemetry + * @return tracing telemetry instance + */ + TracingTelemetry getTracingTelemetry(); + + /** + * Provides metrics telemetry + * @return metrics telemetry instance + */ + MetricsTelemetry getMetricsTelemetry(); + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java new file mode 100644 index 0000000000000..fa3b7fd192f1a --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +/** + * Interface for metrics telemetry providers + */ +public interface MetricsTelemetry { + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/package-info.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/package-info.java new file mode 100644 index 0000000000000..dfe17cc1c11ed --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains metrics related classes + */ +package org.opensearch.telemetry.metrics; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/package-info.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/package-info.java new file mode 100644 index 0000000000000..ad76f5e308bea --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains telemetry related classes + */ +package org.opensearch.telemetry; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java new file mode 100644 index 0000000000000..316edc971913e --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * Base span + */ +public abstract class AbstractSpan implements Span { + + /** + * name of the span + */ + private final String spanName; + /** + * span's parent span + */ + private final Span parentSpan; + + /** + * Base constructor + * @param spanName name of the span + * @param parentSpan span's parent span + */ + protected AbstractSpan(String spanName, Span parentSpan) { + this.spanName = spanName; + this.parentSpan = parentSpan; + } + + @Override + public Span getParentSpan() { + return parentSpan; + } + + @Override + public String getSpanName() { + return spanName; + } + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java new file mode 100644 index 0000000000000..ab9110af7c3ab --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java @@ -0,0 +1,109 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import java.io.Closeable; +import java.io.IOException; + +/** + * + * The default tracer implementation. This class implements the basic logic for span lifecycle and its state management. + * It also handles tracing context propagation between spans. + * + * + */ +public class DefaultTracer implements Tracer { + static final String THREAD_NAME = "th_name"; + + private final TracingTelemetry tracingTelemetry; + private final TracerContextStorage tracerContextStorage; + + /** + * Creates DefaultTracer instance + * + * @param tracingTelemetry tracing telemetry instance + * @param tracerContextStorage storage used for storing current span context + */ + public DefaultTracer(TracingTelemetry tracingTelemetry, TracerContextStorage tracerContextStorage) { + this.tracingTelemetry = tracingTelemetry; + this.tracerContextStorage = tracerContextStorage; + } + + @Override + public Scope startSpan(String spanName) { + Span span = createSpan(spanName, getCurrentSpan()); + setCurrentSpanInContext(span); + addDefaultAttributes(span); + return new ScopeImpl(() -> endSpan(span)); + } + + @Override + public void addSpanAttribute(String key, String value) { + Span currentSpan = getCurrentSpan(); + currentSpan.addAttribute(key, value); + } + + @Override + public void addSpanAttribute(String key, long value) { + Span currentSpan = getCurrentSpan(); + currentSpan.addAttribute(key, value); + } + + @Override + public void addSpanAttribute(String key, double value) { + Span currentSpan = getCurrentSpan(); + currentSpan.addAttribute(key, value); + } + + @Override + public void addSpanAttribute(String key, boolean value) { + Span currentSpan = getCurrentSpan(); + currentSpan.addAttribute(key, value); + } + + @Override + public void addSpanEvent(String event) { + Span currentSpan = getCurrentSpan(); + currentSpan.addEvent(event); + } + + @Override + public void close() throws IOException { + ((Closeable) tracingTelemetry).close(); + } + + // Visible for testing + Span getCurrentSpan() { + return tracerContextStorage.get(TracerContextStorage.CURRENT_SPAN); + } + + private void endSpan(Span span) { + if (span != null) { + span.endSpan(); + setCurrentSpanInContext(span.getParentSpan()); + } + } + + private Span createSpan(String spanName, Span parentSpan) { + return tracingTelemetry.createSpan(spanName, parentSpan); + } + + private void setCurrentSpanInContext(Span span) { + tracerContextStorage.put(TracerContextStorage.CURRENT_SPAN, span); + } + + /** + * Adds default attributes in the span + * @param span the current active span + */ + protected void addDefaultAttributes(Span span) { + span.addAttribute(THREAD_NAME, Thread.currentThread().getName()); + } + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java new file mode 100644 index 0000000000000..52f4eaf648eea --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * An auto-closeable that represents scope of the span. + * It is recommended that you use this class with a try-with-resources block: + */ +public interface Scope extends AutoCloseable { + /** + * No-op Scope implementation + */ + Scope NO_OP = () -> {}; + + /** + * closes the scope + */ + @Override + void close(); +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java new file mode 100644 index 0000000000000..30a7ac7fa90e7 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * Executes the runnable on close + */ +public class ScopeImpl implements Scope { + + private Runnable runnableOnClose; + + /** + * Creates Scope instance + * @param runnableOnClose runnable to execute on scope close + */ + public ScopeImpl(Runnable runnableOnClose) { + this.runnableOnClose = runnableOnClose; + } + + /** + * Executes the runnable to end the scope + */ + @Override + public void close() { + runnableOnClose.run(); + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java new file mode 100644 index 0000000000000..0710b8a22a37f --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * An interface that represents a tracing span. + * Spans are created by the Tracer.startSpan method. + * Span must be ended by calling Tracer.endSpan which internally calls Span's endSpan. + */ +public interface Span { + + /** + * Ends the span + */ + void endSpan(); + + /** + * Returns span's parent span + */ + Span getParentSpan(); + + /** + * Returns the name of the {@link Span} + */ + String getSpanName(); + + /** + * Adds string type attribute in the span + * + * @param key of the attribute + * @param value value of the attribute + */ + void addAttribute(String key, String value); + + /** + * Adds long type attribute in the span + * + * @param key of the attribute + * @param value value of the attribute + */ + void addAttribute(String key, Long value); + + /** + * Adds double type attribute in the span + * + * @param key of the attribute + * @param value value of the attribute + */ + void addAttribute(String key, Double value); + + /** + * Adds boolean type attribute in the span + * + * @param key of the attribute + * @param value value of the attribute + */ + void addAttribute(String key, Boolean value); + + /** + * Adds an event in the span + * + * @param event name of the event + */ + void addEvent(String event); + + /** + * Returns traceId of the span + * @return span's traceId + */ + String getTraceId(); + + /** + * Returns spanId of the span + * @return span's spanId + */ + String getSpanId(); + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java new file mode 100644 index 0000000000000..99d1bd3c93c84 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * Wrapper class to hold reference of Span + */ +public class SpanReference { + + private Span span; + + /** + * Creates the wrapper with given span + * @param span the span object to wrap + */ + public SpanReference(Span span) { + this.span = span; + } + + /** + * Returns the span object + * @return underlying span + */ + public Span getSpan() { + return span; + } + + /** + * Updates the underlying span + * @param span underlying span + */ + public void setSpan(Span span) { + this.span = span; + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java new file mode 100644 index 0000000000000..fcc091eb39c48 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import java.io.Closeable; + +/** + * Tracer is the interface used to create a {@link Span} and interact with current active {@link Span}. + * It automatically handles the context propagation between threads, tasks, nodes etc. + * + * All methods on the Tracer object are multi-thread safe. + */ +public interface Tracer extends Closeable { + + /** + * Starts the {@link Span} with given name + * + * @param spanName span name + * @return scope of the span, must be closed with explicit close or with try-with-resource + */ + Scope startSpan(String spanName); + + /** + * Adds string attribute to the current active {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, String value); + + /** + * Adds long attribute to the current active {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, long value); + + /** + * Adds double attribute to the current active {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, double value); + + /** + * Adds boolean attribute to the current active {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, boolean value); + + /** + * Adds an event to the current active {@link Span}. + * + * @param event event name + */ + void addSpanEvent(String event); +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java new file mode 100644 index 0000000000000..eb93006835332 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * Storage interface used for storing tracing context + * @param key type + * @param value type + */ +public interface TracerContextStorage { + /** + * Key for storing current span + */ + String CURRENT_SPAN = "current_span"; + + /** + * Fetches value corresponding to key + * @param key of the tracing context + * @return value for key + */ + V get(K key); + + /** + * Puts tracing context value with key + * @param key of the tracing context + * @param value of the tracing context + */ + void put(K key, V value); +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java new file mode 100644 index 0000000000000..1152e3aedfa88 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * Interface defining the tracing related context propagation + */ +public interface TracingContextPropagator { + + /** + * Extracts current span from context + * @param props properties + * @return current span + */ + Span extract(Map props); + + /** + * Injects tracing context + * + * @param currentSpan the current active span + * @param setter to add tracing context in map + */ + void inject(Span currentSpan, BiConsumer setter); + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java new file mode 100644 index 0000000000000..16c76bd0cc141 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import java.io.Closeable; + +/** + * Interface for tracing telemetry providers + */ +public interface TracingTelemetry extends Closeable { + + /** + * Creates span with provided arguments + * @param spanName name of the span + * @param parentSpan span's parent span + * @return span instance + */ + Span createSpan(String spanName, Span parentSpan); + + /** + * provides tracing context propagator + * @return tracing context propagator instance + */ + TracingContextPropagator getContextPropagator(); + + /** + * closes the resource + */ + void close(); + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java new file mode 100644 index 0000000000000..18fc60e41e54d --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.noop; + +import org.opensearch.telemetry.tracing.Scope; +import org.opensearch.telemetry.tracing.Tracer; + +/** + * No-op implementation of Tracer + */ +public class NoopTracer implements Tracer { + + /** + * No-op Tracer instance + */ + public static final Tracer INSTANCE = new NoopTracer(); + + private NoopTracer() {} + + @Override + public Scope startSpan(String spanName) { + return Scope.NO_OP; + } + + /** + * @param key attribute key + * @param value attribute value + */ + @Override + public void addSpanAttribute(String key, String value) { + + } + + /** + * @param key attribute key + * @param value attribute value + */ + @Override + public void addSpanAttribute(String key, long value) { + + } + + /** + * @param key attribute key + * @param value attribute value + */ + @Override + public void addSpanAttribute(String key, double value) { + + } + + /** + * @param key attribute key + * @param value attribute value + */ + @Override + public void addSpanAttribute(String key, boolean value) { + + } + + @Override + public void addSpanEvent(String event) { + + } + + @Override + public void close() { + + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/package-info.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/package-info.java new file mode 100644 index 0000000000000..b9d83e7bc7275 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains No-op implementations + */ +package org.opensearch.telemetry.tracing.noop; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/package-info.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/package-info.java new file mode 100644 index 0000000000000..66898bd58b753 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains tracing related classes + */ +package org.opensearch.telemetry.tracing; diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java new file mode 100644 index 0000000000000..f0e8f3c2e2344 --- /dev/null +++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.junit.Assert; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; + +public class DefaultTracerTests extends OpenSearchTestCase { + + private TracingTelemetry mockTracingTelemetry; + private TracerContextStorage mockTracerContextStorage; + private Span mockSpan; + private Span mockParentSpan; + + @Override + public void setUp() throws Exception { + super.setUp(); + setupMocks(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + public void testCreateSpan() { + DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + + defaultTracer.startSpan("span_name"); + + Assert.assertEquals("span_name", defaultTracer.getCurrentSpan().getSpanName()); + } + + public void testEndSpanByClosingScope() { + DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + try (Scope scope = defaultTracer.startSpan("span_name")) { + verify(mockTracerContextStorage).put(TracerContextStorage.CURRENT_SPAN, mockSpan); + } + verify(mockTracerContextStorage).put(TracerContextStorage.CURRENT_SPAN, mockParentSpan); + } + + public void testAddSpanAttributeString() { + Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + defaultTracer.startSpan("span_name"); + + defaultTracer.addSpanAttribute("key", "value"); + + verify(mockSpan).addAttribute("key", "value"); + } + + public void testAddSpanAttributeLong() { + Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + defaultTracer.startSpan("span_name"); + + defaultTracer.addSpanAttribute("key", 1L); + + verify(mockSpan).addAttribute("key", 1L); + } + + public void testAddSpanAttributeDouble() { + Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + defaultTracer.startSpan("span_name"); + + defaultTracer.addSpanAttribute("key", 1.0); + + verify(mockSpan).addAttribute("key", 1.0); + } + + public void testAddSpanAttributeBoolean() { + Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + defaultTracer.startSpan("span_name"); + + defaultTracer.addSpanAttribute("key", true); + + verify(mockSpan).addAttribute("key", true); + } + + public void testAddEvent() { + Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + defaultTracer.startSpan("span_name"); + + defaultTracer.addSpanEvent("eventName"); + + verify(mockSpan).addEvent("eventName"); + } + + public void testClose() throws IOException { + Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); + + defaultTracer.close(); + + verify(mockTracingTelemetry).close(); + } + + @SuppressWarnings("unchecked") + private void setupMocks() { + mockTracingTelemetry = mock(TracingTelemetry.class); + mockSpan = mock(Span.class); + mockParentSpan = mock(Span.class); + mockTracerContextStorage = mock(TracerContextStorage.class); + when(mockSpan.getSpanName()).thenReturn("span_name"); + when(mockSpan.getSpanId()).thenReturn("span_id"); + when(mockSpan.getTraceId()).thenReturn("trace_id"); + when(mockSpan.getParentSpan()).thenReturn(mockParentSpan); + when(mockParentSpan.getSpanId()).thenReturn("parent_span_id"); + when(mockParentSpan.getTraceId()).thenReturn("trace_id"); + when(mockTracerContextStorage.get(TracerContextStorage.CURRENT_SPAN)).thenReturn(mockParentSpan, mockSpan); + when(mockTracingTelemetry.createSpan("span_name", mockParentSpan)).thenReturn(mockSpan); + } +} diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle new file mode 100644 index 0000000000000..7a56621be5f1e --- /dev/null +++ b/plugins/telemetry-otel/build.gradle @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.apache.tools.ant.taskdefs.condition.Os +import org.opensearch.gradle.Architecture +import org.opensearch.gradle.OS +import org.opensearch.gradle.info.BuildParams + +opensearchplugin { + description 'Opentelemetry based telemetry implementation.' + classname 'org.opensearch.telemetry.OTelTelemetryPlugin' + hasClientJar = true +} + +dependencies { + api project(":libs:opensearch-telemetry") + api "io.opentelemetry:opentelemetry-api:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-context:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-sdk:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-sdk-common:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-sdk-trace:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-sdk-metrics:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-exporter-logging:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha" + api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}-alpha" + api "io.opentelemetry:opentelemetry-api-logs:${versions.opentelemetry}-alpha" +} + + +thirdPartyAudit { + ignoreViolations( + 'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueConsumerIndexField', + 'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerIndexField', + 'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.opentelemetry.internal.shaded.jctools.util.UnsafeAccess', + 'io.opentelemetry.internal.shaded.jctools.util.UnsafeRefArrayAccess' + ) + + ignoreMissingClasses( + 'io.opentelemetry.api.events.EventEmitter', + 'io.opentelemetry.api.events.EventEmitterBuilder', + 'io.opentelemetry.api.events.EventEmitterProvider', + 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleHistogramBuilder', + 'io.opentelemetry.extension.incubator.metrics.ExtendedLongHistogramBuilder', + 'io.opentelemetry.extension.incubator.metrics.HistogramAdviceConfigurer', + 'io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties', + 'io.opentelemetry.sdk.autoconfigure.spi.logs.ConfigurableLogRecordExporterProvider', + 'io.opentelemetry.sdk.autoconfigure.spi.metrics.ConfigurableMetricExporterProvider', + 'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider' + ) +} diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..da3abcc8f70d2 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 @@ -0,0 +1 @@ +7ee1ccca95155e4640094ba8dfbd0bb8c1709c83 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-api-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 new file mode 100644 index 0000000000000..2c233d785dcb2 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 @@ -0,0 +1 @@ +1b0b6c1a20da0f841634d4f736e331aa4871a4db \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..01d9fd732249b --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 @@ -0,0 +1 @@ +42991f523a7a10761213e2f11633c67c8beaed88 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-context-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-context-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-context-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..ef07e4cb81e34 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 @@ -0,0 +1 @@ +1b932170774da5e766440fa058d879f68fe2c5dd \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..dc9946de3b160 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 @@ -0,0 +1 @@ +79a86f258ede8625627e8fbdff07d1149c88a8e6 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..2bd3e60a1faf6 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 @@ -0,0 +1 @@ +b42359d2232f8d802d55153be5330b1d9e21ee15 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 new file mode 100644 index 0000000000000..90bb8202c4c9d --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 @@ -0,0 +1 @@ +a8abeaee240291cce9067f07569f151d11a6275a \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..62396a603423f --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 @@ -0,0 +1 @@ +8c4af22d7d92a3a79714be3f79724b0ab774ba9e \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 new file mode 100644 index 0000000000000..0fcebee353105 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 @@ -0,0 +1 @@ +fcc5785b2cf2be897f31b927e24b53e46e377388 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 new file mode 100644 index 0000000000000..47c7ece8c9f6c --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 @@ -0,0 +1 @@ +1f4f963673f8209208f868666cd43e79b9a2dd15 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-semconv-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-semconv-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java new file mode 100644 index 0000000000000..1c38c9dc8d6be --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java @@ -0,0 +1,92 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.TelemetryPlugin; +import org.opensearch.telemetry.metrics.MetricsTelemetry; +import org.opensearch.telemetry.tracing.OTelResourceProvider; +import org.opensearch.telemetry.tracing.OTelTelemetry; +import org.opensearch.telemetry.tracing.OTelTracingTelemetry; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +/** + * Telemetry plugin based on Otel + */ +public class OTelTelemetryPlugin extends Plugin implements TelemetryPlugin { + + static final String OTEL_TRACER_NAME = "otel"; + + /** + * span exporter batch size + */ + public static final Setting TRACER_EXPORTER_BATCH_SIZE_SETTING = Setting.intSetting( + "telemetry.otel.tracer.exporter.batch_size", + 512, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** + * span exporter max queue size + */ + public static final Setting TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING = Setting.intSetting( + "telemetry.otel.tracer.exporter.max_queue_size", + 2048, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** + * span exporter delay in seconds + */ + public static final Setting TRACER_EXPORTER_DELAY_SETTING = Setting.timeSetting( + "telemetry.otel.tracer.exporter.delay", + TimeValue.timeValueSeconds(2), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private final Settings settings; + + /** + * Creates Otel plugin + * @param settings cluster settings + */ + public OTelTelemetryPlugin(Settings settings) { + this.settings = settings; + } + + @Override + public List> getSettings() { + return Arrays.asList(TRACER_EXPORTER_BATCH_SIZE_SETTING, TRACER_EXPORTER_DELAY_SETTING, TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING); + } + + @Override + public Optional getTelemetry(TelemetrySettings settings) { + return Optional.of(telemetry()); + } + + @Override + public String getName() { + return OTEL_TRACER_NAME; + } + + private Telemetry telemetry() { + return new OTelTelemetry(new OTelTracingTelemetry(OTelResourceProvider.get(settings)), new MetricsTelemetry() { + }); + } + +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/package-info.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/package-info.java new file mode 100644 index 0000000000000..4545f0ef5990e --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for telemetry. + */ +package org.opensearch.telemetry; diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelPropagatedSpan.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelPropagatedSpan.java new file mode 100644 index 0000000000000..5aa1069e60367 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelPropagatedSpan.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +/** + * Propagated span through context propagation + */ +public class OTelPropagatedSpan extends OTelSpan { + + /** + * Creates OTelPropagatedSpan + * @param span otel propagated span + */ + public OTelPropagatedSpan(io.opentelemetry.api.trace.Span span) { + super(null, span, null); + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java new file mode 100644 index 0000000000000..04bade9ec942a --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import org.opensearch.common.settings.Settings; + +import java.util.concurrent.TimeUnit; + +import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_BATCH_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_DELAY_SETTING; +import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING; + +/** + * This class encapsulates all OpenTelemetry related resources + */ +public final class OTelResourceProvider { + private OTelResourceProvider() {} + + /** + * Creates OpenTelemetry instance with default configuration + * @param settings cluster settings + * @return OpenTelemetry instance + */ + public static OpenTelemetry get(Settings settings) { + return get( + settings, + new LoggingSpanExporter(), + ContextPropagators.create(W3CTraceContextPropagator.getInstance()), + Sampler.alwaysOn() + ); + } + + /** + * Creates OpenTelemetry instance with provided configuration + * @param settings cluster settings + * @param spanExporter span exporter instance + * @param contextPropagators context propagator instance + * @param sampler sampler instance + * @return Opentelemetry instance + */ + public static OpenTelemetry get(Settings settings, SpanExporter spanExporter, ContextPropagators contextPropagators, Sampler sampler) { + Resource resource = Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenSearch")); + SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(spanProcessor(settings, spanExporter)) + .setResource(resource) + .setSampler(sampler) + .build(); + + return OpenTelemetrySdk.builder().setTracerProvider(sdkTracerProvider).setPropagators(contextPropagators).buildAndRegisterGlobal(); + } + + private static BatchSpanProcessor spanProcessor(Settings settings, SpanExporter spanExporter) { + return BatchSpanProcessor.builder(spanExporter) + .setScheduleDelay(TRACER_EXPORTER_DELAY_SETTING.get(settings).getSeconds(), TimeUnit.SECONDS) + .setMaxExportBatchSize(TRACER_EXPORTER_BATCH_SIZE_SETTING.get(settings)) + .setMaxQueueSize(TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING.get(settings)) + .build(); + } + +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java new file mode 100644 index 0000000000000..23a2d9baa3e6e --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.trace.Span; + +/** + * Default implementation of {@link Span} using Otel span. It keeps a reference of OpenTelemetry Span and handles span + * lifecycle management by delegating calls to it. + */ +class OTelSpan extends AbstractSpan { + + private final Span delegateSpan; + + public OTelSpan(String spanName, Span span, org.opensearch.telemetry.tracing.Span parentSpan) { + super(spanName, parentSpan); + this.delegateSpan = span; + } + + @Override + public void endSpan() { + delegateSpan.end(); + } + + @Override + public void addAttribute(String key, String value) { + delegateSpan.setAttribute(key, value); + } + + @Override + public void addAttribute(String key, Long value) { + delegateSpan.setAttribute(key, value); + } + + @Override + public void addAttribute(String key, Double value) { + delegateSpan.setAttribute(key, value); + } + + @Override + public void addAttribute(String key, Boolean value) { + delegateSpan.setAttribute(key, value); + } + + @Override + public void addEvent(String event) { + delegateSpan.addEvent(event); + } + + @Override + public String getTraceId() { + return delegateSpan.getSpanContext().getTraceId(); + } + + @Override + public String getSpanId() { + return delegateSpan.getSpanContext().getSpanId(); + } + + io.opentelemetry.api.trace.Span getDelegateSpan() { + return delegateSpan; + } + +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java new file mode 100644 index 0000000000000..282fabd43346b --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.metrics.MetricsTelemetry; + +/** + * Otel implementation of Telemetry + */ +public class OTelTelemetry implements Telemetry { + + private final TracingTelemetry tracingTelemetry; + private final MetricsTelemetry metricsTelemetry; + + /** + * Creates Telemetry instance + * @param tracingTelemetry tracing telemetry + * @param metricsTelemetry metrics telemetry + */ + public OTelTelemetry(TracingTelemetry tracingTelemetry, MetricsTelemetry metricsTelemetry) { + this.tracingTelemetry = tracingTelemetry; + this.metricsTelemetry = metricsTelemetry; + } + + @Override + public TracingTelemetry getTracingTelemetry() { + return tracingTelemetry; + } + + @Override + public MetricsTelemetry getMetricsTelemetry() { + return metricsTelemetry; + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagator.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagator.java new file mode 100644 index 0000000000000..15609b39b6b94 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagator.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.TextMapGetter; +import io.opentelemetry.context.propagation.TextMapSetter; + +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * Otel implementation of TracingContextPropagator + */ +public class OTelTracingContextPropagator implements TracingContextPropagator { + + private final OpenTelemetry openTelemetry; + + /** + * Creates OTelTracingContextPropagator instance + * @param openTelemetry Otel OpenTelemetry instance + */ + public OTelTracingContextPropagator(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + + @Override + public Span extract(Map props) { + Context context = openTelemetry.getPropagators().getTextMapPropagator().extract(Context.current(), props, TEXT_MAP_GETTER); + if (context != null) { + io.opentelemetry.api.trace.Span span = io.opentelemetry.api.trace.Span.fromContext(context); + return new OTelPropagatedSpan(span); + } + return null; + } + + @Override + public void inject(Span currentSpan, BiConsumer setter) { + openTelemetry.getPropagators().getTextMapPropagator().inject(context((OTelSpan) currentSpan), setter, TEXT_MAP_SETTER); + + } + + private static Context context(OTelSpan oTelSpan) { + return Context.current().with(io.opentelemetry.api.trace.Span.wrap(oTelSpan.getDelegateSpan().getSpanContext())); + } + + private static final TextMapSetter> TEXT_MAP_SETTER = (carrier, key, value) -> { + if (carrier != null) { + carrier.accept(key, value); + } + }; + + private static final TextMapGetter> TEXT_MAP_GETTER = new TextMapGetter<>() { + @Override + public Iterable keys(Map headers) { + return headers.keySet(); + } + + @Override + public String get(Map headers, String key) { + if (headers != null && headers.containsKey(key)) { + return headers.get(key); + } + return null; + } + }; + +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java new file mode 100644 index 0000000000000..8a0034e098461 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.Context; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.Closeable; +import java.io.IOException; + +/** + * OTel based Telemetry provider + */ +public class OTelTracingTelemetry implements TracingTelemetry { + + private static final Logger logger = LogManager.getLogger(OTelTracingTelemetry.class); + + private final OpenTelemetry openTelemetry; + private final io.opentelemetry.api.trace.Tracer otelTracer; + + /** + * Creates OTel based Telemetry + * @param openTelemetry OpenTelemetry instance + */ + public OTelTracingTelemetry(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + this.otelTracer = openTelemetry.getTracer("os-tracer"); + + } + + @Override + public void close() { + try { + ((Closeable) openTelemetry).close(); + } catch (IOException e) { + logger.warn("Error while closing Opentelemetry", e); + } + } + + @Override + public Span createSpan(String spanName, Span parentSpan) { + return createOtelSpan(spanName, parentSpan); + } + + @Override + public TracingContextPropagator getContextPropagator() { + return new OTelTracingContextPropagator(openTelemetry); + } + + private Span createOtelSpan(String spanName, Span parentSpan) { + io.opentelemetry.api.trace.Span otelSpan = otelSpan(spanName, parentSpan); + return new OTelSpan(spanName, otelSpan, parentSpan); + } + + io.opentelemetry.api.trace.Span otelSpan(String spanName, Span parentOTelSpan) { + return parentOTelSpan == null || !(parentOTelSpan instanceof OTelSpan) + ? otelTracer.spanBuilder(spanName).startSpan() + : otelTracer.spanBuilder(spanName).setParent(Context.current().with(((OTelSpan) parentOTelSpan).getDelegateSpan())).startSpan(); + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/package-info.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/package-info.java new file mode 100644 index 0000000000000..4ac1e4c212c81 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.tracing; diff --git a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..0f556121915bb --- /dev/null +++ b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { +}; + + diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java new file mode 100644 index 0000000000000..c6ffba04ac285 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.telemetry.tracing.OTelTracingTelemetry; +import org.opensearch.telemetry.tracing.TracingTelemetry; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Optional; + +import static org.opensearch.telemetry.OTelTelemetryPlugin.OTEL_TRACER_NAME; +import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_BATCH_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_DELAY_SETTING; +import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING; + +public class OTelTelemetryPluginTests extends OpenSearchTestCase { + + public void testGetTelemetry() { + Set> allTracerSettings = new HashSet<>(); + ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add)); + Settings settings = Settings.builder().build(); + OTelTelemetryPlugin oTelTracerModulePlugin = new OTelTelemetryPlugin(settings); + Optional tracer = oTelTracerModulePlugin.getTelemetry(null); + + assertEquals(OTEL_TRACER_NAME, oTelTracerModulePlugin.getName()); + TracingTelemetry tracingTelemetry = tracer.get().getTracingTelemetry(); + assertTrue(tracingTelemetry instanceof OTelTracingTelemetry); + assertEquals( + Arrays.asList(TRACER_EXPORTER_BATCH_SIZE_SETTING, TRACER_EXPORTER_DELAY_SETTING, TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING), + oTelTracerModulePlugin.getSettings() + ); + tracingTelemetry.close(); + + } + +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanTests.java new file mode 100644 index 0000000000000..ac849e620673a --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanTests.java @@ -0,0 +1,92 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import org.opensearch.test.OpenSearchTestCase; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OTelSpanTests extends OpenSearchTestCase { + + private static final String TRACE_ID = "4aa59968f31dcbff7807741afa9d7d62"; + private static final String SPAN_ID = "bea205cd25756b5e"; + + public void testEndSpanTest() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + oTelSpan.endSpan(); + verify(mockSpan).end(); + } + + public void testAddAttributeString() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + oTelSpan.addAttribute("key", "value"); + + verify(mockSpan).setAttribute("key", "value"); + } + + public void testAddAttributeLong() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + oTelSpan.addAttribute("key", 1L); + + verify(mockSpan).setAttribute("key", 1L); + } + + public void testAddAttributeDouble() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + oTelSpan.addAttribute("key", 1.0); + + verify(mockSpan).setAttribute("key", 1.0); + } + + public void testAddAttributeBoolean() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + oTelSpan.addAttribute("key", true); + + verify(mockSpan).setAttribute("key", true); + } + + public void testAddEvent() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + oTelSpan.addEvent("eventName"); + + verify(mockSpan).addEvent("eventName"); + } + + public void testGetTraceId() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + + assertEquals(TRACE_ID, oTelSpan.getTraceId()); + } + + public void testGetSpanId() { + Span mockSpan = getMockSpan(); + OTelSpan oTelSpan = new OTelSpan("spanName", mockSpan, null); + + assertEquals(SPAN_ID, oTelSpan.getSpanId()); + } + + private Span getMockSpan() { + Span mockSpan = mock(Span.class); + when(mockSpan.getSpanContext()).thenReturn(SpanContext.create(TRACE_ID, SPAN_ID, TraceFlags.getDefault(), TraceState.getDefault())); + return mockSpan; + } +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagatorTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagatorTests.java new file mode 100644 index 0000000000000..1f76b0b9def18 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingContextPropagatorTests.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OTelTracingContextPropagatorTests extends OpenSearchTestCase { + + private static final String TRACE_ID = "4aa59968f31dcbff7807741afa9d7d62"; + private static final String SPAN_ID = "bea205cd25756b5e"; + + public void testAddTracerContextToHeader() { + Span mockSpan = mock(Span.class); + when(mockSpan.getSpanContext()).thenReturn(SpanContext.create(TRACE_ID, SPAN_ID, TraceFlags.getDefault(), TraceState.getDefault())); + OTelSpan span = new OTelSpan("spanName", mockSpan, null); + Map requestHeaders = new HashMap<>(); + OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); + when(mockOpenTelemetry.getPropagators()).thenReturn(ContextPropagators.create(W3CTraceContextPropagator.getInstance())); + TracingContextPropagator tracingContextPropagator = new OTelTracingContextPropagator(mockOpenTelemetry); + + tracingContextPropagator.inject(span, (key, value) -> requestHeaders.put(key, value)); + assertEquals("00-" + TRACE_ID + "-" + SPAN_ID + "-00", requestHeaders.get("traceparent")); + } + + public void testExtractTracerContextFromHeader() { + Map requestHeaders = new HashMap<>(); + requestHeaders.put("traceparent", "00-" + TRACE_ID + "-" + SPAN_ID + "-00"); + OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); + when(mockOpenTelemetry.getPropagators()).thenReturn(ContextPropagators.create(W3CTraceContextPropagator.getInstance())); + TracingContextPropagator tracingContextPropagator = new OTelTracingContextPropagator(mockOpenTelemetry); + org.opensearch.telemetry.tracing.Span span = tracingContextPropagator.extract(requestHeaders); + assertEquals(TRACE_ID, span.getTraceId()); + assertEquals(SPAN_ID, span.getSpanId()); + } +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java new file mode 100644 index 0000000000000..7dec7824b9790 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.Tracer; +import org.opensearch.test.OpenSearchTestCase; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class OTelTracingTelemetryTests extends OpenSearchTestCase { + + public void testCreateSpanWithoutParent() { + OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); + Tracer mockTracer = mock(Tracer.class); + when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + SpanBuilder mockSpanBuilder = mock(SpanBuilder.class); + when(mockTracer.spanBuilder("span_name")).thenReturn(mockSpanBuilder); + when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class)); + + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + Span span = tracingTelemetry.createSpan("span_name", null); + + verify(mockSpanBuilder, never()).setParent(any()); + assertNull(span.getParentSpan()); + } + + public void testCreateSpanWithParent() { + OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); + Tracer mockTracer = mock(Tracer.class); + when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + SpanBuilder mockSpanBuilder = mock(SpanBuilder.class); + when(mockTracer.spanBuilder("span_name")).thenReturn(mockSpanBuilder); + when(mockSpanBuilder.setParent(any())).thenReturn(mockSpanBuilder); + when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class)); + + Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null); + + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + Span span = tracingTelemetry.createSpan("span_name", parentSpan); + + verify(mockSpanBuilder).setParent(any()); + assertNotNull(span.getParentSpan()); + assertEquals("parent_span", span.getParentSpan().getSpanName()); + } + + public void testGetContextPropagator() { + OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); + Tracer mockTracer = mock(Tracer.class); + when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + + assertTrue(tracingTelemetry.getContextPropagator() instanceof OTelTracingContextPropagator); + } + +} diff --git a/server/build.gradle b/server/build.gradle index 2738e4351a109..ab67eabe76d0c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -102,6 +102,8 @@ dependencies { api project(':libs:opensearch-secure-sm') api project(':libs:opensearch-x-content') api project(":libs:opensearch-geo") + api project(":libs:opensearch-telemetry") + compileOnly project(':libs:opensearch-plugin-classloader') testRuntimeOnly project(':libs:opensearch-plugin-classloader') diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index fe1d292dbd8f6..d70ea16cf5fdd 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -52,6 +52,7 @@ import org.opensearch.tasks.TaskManager; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.tasks.consumer.TopNSearchTasksLogger; +import org.opensearch.telemetry.TelemetrySettings; import org.opensearch.watcher.ResourceWatcherService; import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; import org.opensearch.action.admin.indices.close.TransportCloseIndexAction; @@ -675,6 +676,8 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING ), List.of(FeatureFlags.CONCURRENT_SEGMENT_SEARCH), - List.of(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + List.of(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING), + List.of(FeatureFlags.TELEMETRY), + List.of(TelemetrySettings.TRACER_ENABLED_SETTING) ); } diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 19a5808579d50..dae66c79c63ec 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -39,7 +39,8 @@ protected FeatureFlagSettings( FeatureFlags.EXTENSIONS_SETTING, FeatureFlags.IDENTITY_SETTING, FeatureFlags.SEARCH_PIPELINE_SETTING, - FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING + FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING, + FeatureFlags.TELEMETRY_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index a26b4006bb31e..2b4dab616d00f 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -63,6 +63,11 @@ public class FeatureFlags { */ public static final String CONCURRENT_SEGMENT_SEARCH = "opensearch.experimental.feature.concurrent_segment_search.enabled"; + /** + * Gates the functionality of telemetry framework. + */ + public static final String TELEMETRY = "opensearch.experimental.feature.telemetry.enabled"; + /** * Should store the settings from opensearch.yml. */ @@ -105,6 +110,8 @@ public static boolean isEnabled(String featureFlagName) { public static final Setting IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope); + public static final Setting TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope); + public static final Setting CONCURRENT_SEGMENT_SEARCH_SETTING = Setting.boolSetting( CONCURRENT_SEGMENT_SEARCH, false, diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java index 5b9a77c75dddb..025fb7a36b684 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java @@ -45,6 +45,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.http.HttpTransportSettings; import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskThreadContextStatePropagator; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -56,7 +57,9 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; @@ -66,7 +69,6 @@ import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE; -import static org.opensearch.tasks.TaskResourceTrackingService.TASK_ID; /** * A ThreadContext is a map of string headers and a transient map of keyed objects that are associated with @@ -113,6 +115,7 @@ public final class ThreadContext implements Writeable { private final ThreadLocal threadLocal; private final int maxWarningHeaderCount; private final long maxWarningHeaderSize; + private final List propagators; /** * Creates a new ThreadContext instance @@ -123,6 +126,15 @@ public ThreadContext(Settings settings) { this.threadLocal = ThreadLocal.withInitial(() -> DEFAULT_CONTEXT); this.maxWarningHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); this.maxWarningHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); + this.propagators = new CopyOnWriteArrayList<>(List.of(new TaskThreadContextStatePropagator())); + } + + public void registerThreadContextStatePropagator(final ThreadContextStatePropagator propagator) { + propagators.add(Objects.requireNonNull(propagator)); + } + + public void unregisterThreadContextStatePropagator(final ThreadContextStatePropagator propagator) { + propagators.remove(Objects.requireNonNull(propagator)); } /** @@ -147,8 +159,9 @@ public StoredContext stashContext() { ); } - if (context.transientHeaders.containsKey(TASK_ID)) { - threadContextStruct = threadContextStruct.putTransient(TASK_ID, context.transientHeaders.get(TASK_ID)); + final Map transientHeaders = propagateTransients(context.transientHeaders); + if (!transientHeaders.isEmpty()) { + threadContextStruct = threadContextStruct.putTransient(transientHeaders); } threadLocal.set(threadContextStruct); @@ -166,7 +179,10 @@ public StoredContext stashContext() { */ public Writeable captureAsWriteable() { final ThreadContextStruct context = threadLocal.get(); - return out -> context.writeTo(out, defaultHeader); + return out -> { + final Map propagatedHeaders = propagateHeaders(context.transientHeaders); + context.writeTo(out, defaultHeader, propagatedHeaders); + }; } /** @@ -224,17 +240,24 @@ public StoredContext newStoredContext(boolean preserveResponseHeaders) { */ public StoredContext newStoredContext(boolean preserveResponseHeaders, Collection transientHeadersToClear) { final ThreadContextStruct originalContext = threadLocal.get(); + final Map newTransientHeaders = new HashMap<>(originalContext.transientHeaders); + + boolean transientHeadersModified = false; + final Map transientHeaders = propagateTransients(originalContext.transientHeaders); + if (!transientHeaders.isEmpty()) { + newTransientHeaders.putAll(transientHeaders); + transientHeadersModified = true; + } + // clear specific transient headers from the current context - Map newTransientHeaders = null; for (String transientHeaderToClear : transientHeadersToClear) { - if (originalContext.transientHeaders.containsKey(transientHeaderToClear)) { - if (newTransientHeaders == null) { - newTransientHeaders = new HashMap<>(originalContext.transientHeaders); - } + if (newTransientHeaders.containsKey(transientHeaderToClear)) { newTransientHeaders.remove(transientHeaderToClear); + transientHeadersModified = true; } } - if (newTransientHeaders != null) { + + if (transientHeadersModified == true) { ThreadContextStruct threadContextStruct = new ThreadContextStruct( originalContext.requestHeaders, originalContext.responseHeaders, @@ -246,6 +269,7 @@ public StoredContext newStoredContext(boolean preserveResponseHeaders, Collectio } // this is the context when this method returns final ThreadContextStruct newContext = threadLocal.get(); + return () -> { if (preserveResponseHeaders && threadLocal.get() != newContext) { threadLocal.set(originalContext.putResponseHeaders(threadLocal.get().responseHeaders)); @@ -294,7 +318,9 @@ public Supplier wrapRestorable(StoredContext storedContext) { @Override public void writeTo(StreamOutput out) throws IOException { - threadLocal.get().writeTo(out, defaultHeader); + final ThreadContextStruct context = threadLocal.get(); + final Map propagatedHeaders = propagateHeaders(context.transientHeaders); + context.writeTo(out, defaultHeader, propagatedHeaders); } /** @@ -522,6 +548,18 @@ public static Map buildDefaultHeaders(Settings settings) { } } + private Map propagateTransients(Map source) { + final Map transients = new HashMap<>(); + propagators.forEach(p -> transients.putAll(p.transients(source))); + return transients; + } + + private Map propagateHeaders(Map source) { + final Map headers = new HashMap<>(); + propagators.forEach(p -> headers.putAll(p.headers(source))); + return headers; + } + private static final class ThreadContextStruct { private static final ThreadContextStruct EMPTY = new ThreadContextStruct( @@ -695,6 +733,14 @@ private ThreadContextStruct putResponse( return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext, newWarningHeaderSize); } + private ThreadContextStruct putTransient(Map values) { + Map newTransient = new HashMap<>(this.transientHeaders); + for (Map.Entry entry : values.entrySet()) { + putSingleHeader(entry.getKey(), entry.getValue(), newTransient); + } + return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient, isSystemContext); + } + private ThreadContextStruct putTransient(String key, Object value) { Map newTransient = new HashMap<>(this.transientHeaders); putSingleHeader(key, value, newTransient); @@ -709,13 +755,15 @@ private ThreadContextStruct copyHeaders(Iterable> head return putHeaders(newHeaders); } - private void writeTo(StreamOutput out, Map defaultHeaders) throws IOException { + private void writeTo(StreamOutput out, Map defaultHeaders, Map propagatedHeaders) + throws IOException { final Map requestHeaders; - if (defaultHeaders.isEmpty()) { + if (defaultHeaders.isEmpty() && propagatedHeaders.isEmpty()) { requestHeaders = this.requestHeaders; } else { requestHeaders = new HashMap<>(defaultHeaders); requestHeaders.putAll(this.requestHeaders); + requestHeaders.putAll(propagatedHeaders); } out.writeVInt(requestHeaders.size()); diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContextStatePropagator.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContextStatePropagator.java new file mode 100644 index 0000000000000..b3fc79c5446db --- /dev/null +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContextStatePropagator.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util.concurrent; + +import java.util.Map; + +/** + * The propagator for {@link ThreadContext} that helps to carry-over the state from one + * thread to another (tasks, tracing context, etc). + */ +public interface ThreadContextStatePropagator { + /** + * Returns the list of transient headers that needs to be propagated from current context to new thread context. + * @param source current context transient headers + * @return the list of transient headers that needs to be propagated from current context to new thread context + */ + Map transients(Map source); + + /** + * Returns the list of request headers that needs to be propagated from current context to request. + * @param source current context headers + * @return the list of request headers that needs to be propagated from current context to request + */ + Map headers(Map source); +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 688f2d05b203b..3742c817118da 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -56,6 +56,8 @@ import org.opensearch.monitor.fs.FsProbe; import org.opensearch.plugins.ExtensionAwarePlugin; import org.opensearch.plugins.SearchPipelinePlugin; +import org.opensearch.telemetry.tracing.NoopTracerFactory; +import org.opensearch.telemetry.tracing.TracerFactory; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; import org.opensearch.search.pipeline.SearchPipelineService; @@ -65,6 +67,8 @@ import org.opensearch.tasks.consumer.TopNSearchTasksLogger; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; +import org.opensearch.telemetry.TelemetryModule; +import org.opensearch.telemetry.TelemetrySettings; import org.opensearch.watcher.ResourceWatcherService; import org.opensearch.core.Assertions; import org.opensearch.Build; @@ -194,6 +198,7 @@ import org.opensearch.plugins.ScriptPlugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.plugins.SystemIndexPlugin; +import org.opensearch.plugins.TelemetryPlugin; import org.opensearch.repositories.RepositoriesModule; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; @@ -254,6 +259,7 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.common.util.FeatureFlags.SEARCH_PIPELINE; +import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; @@ -373,6 +379,7 @@ public static class DiscoverySettings { private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; + private final TracerFactory tracerFactory; final NamedWriteableRegistry namedWriteableRegistry; private final AtomicReference runnableTaskListener; private FileCache fileCache; @@ -1021,6 +1028,16 @@ protected Node( searchModule.getIndexSearcherExecutor(threadPool) ); + if (FeatureFlags.isEnabled(TELEMETRY)) { + final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings()); + List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class); + TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings); + tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext()); + } else { + tracerFactory = new NoopTracerFactory(); + } + resourcesToClose.add(tracerFactory::close); + final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) .stream() .map( @@ -1126,6 +1143,7 @@ protected Node( b.bind(FsHealthService.class).toInstance(fsHealthService); b.bind(SystemIndices.class).toInstance(systemIndices); b.bind(IdentityService.class).toInstance(identityService); + b.bind(TracerFactory.class).toInstance(this.tracerFactory); }); injector = modules.createInjector(); @@ -1481,6 +1499,9 @@ public synchronized void close() throws IOException { toClose.add(() -> stopWatch.stop().start("node_environment")); toClose.add(injector.getInstance(NodeEnvironment.class)); toClose.add(stopWatch::stop); + if (FeatureFlags.isEnabled(TELEMETRY)) { + toClose.add(() -> injector.getInstance(TracerFactory.class)); + } if (logger.isTraceEnabled()) { toClose.add(() -> logger.trace("Close times for each service:\n{}", stopWatch.prettyPrint())); diff --git a/server/src/main/java/org/opensearch/plugins/TelemetryPlugin.java b/server/src/main/java/org/opensearch/plugins/TelemetryPlugin.java new file mode 100644 index 0000000000000..33dc9b7a0c843 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/TelemetryPlugin.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; + +import java.util.Optional; + +/** + * Plugin for extending telemetry related classes + */ +public interface TelemetryPlugin { + + Optional getTelemetry(TelemetrySettings settings); + + String getName(); + +} diff --git a/server/src/main/java/org/opensearch/tasks/TaskThreadContextStatePropagator.java b/server/src/main/java/org/opensearch/tasks/TaskThreadContextStatePropagator.java new file mode 100644 index 0000000000000..ed111b34f048f --- /dev/null +++ b/server/src/main/java/org/opensearch/tasks/TaskThreadContextStatePropagator.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tasks; + +import org.opensearch.common.util.concurrent.ThreadContextStatePropagator; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.tasks.TaskResourceTrackingService.TASK_ID; + +/** + * Propagates TASK_ID across thread contexts + */ +public class TaskThreadContextStatePropagator implements ThreadContextStatePropagator { + @Override + public Map transients(Map source) { + final Map transients = new HashMap<>(); + + if (source.containsKey(TASK_ID)) { + transients.put(TASK_ID, source.get(TASK_ID)); + } + + return transients; + } + + @Override + public Map headers(Map source) { + return Collections.emptyMap(); + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/TelemetryModule.java b/server/src/main/java/org/opensearch/telemetry/TelemetryModule.java new file mode 100644 index 0000000000000..604c111b0720c --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/TelemetryModule.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import org.opensearch.plugins.TelemetryPlugin; + +import java.util.List; +import java.util.Optional; + +/** + * A module for loading classes for telemetry + * + * @opensearch.internal + */ +public class TelemetryModule { + + private Telemetry telemetry; + + public TelemetryModule(List telemetryPlugins, TelemetrySettings telemetrySettings) { + + for (TelemetryPlugin telemetryPlugin : telemetryPlugins) { + Optional telemetry = telemetryPlugin.getTelemetry(telemetrySettings); + if (telemetry.isPresent()) { + registerTelemetry(telemetry.get()); + } + } + } + + public Optional getTelemetry() { + return Optional.ofNullable(telemetry); + } + + private void registerTelemetry(Telemetry factory) { + if (telemetry == null) { + telemetry = factory; + } else { + throw new IllegalArgumentException("Cannot register more than one telemetry"); + } + } + +} diff --git a/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java b/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java new file mode 100644 index 0000000000000..7c9e0d5ac8097 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; + +/** + * Wrapper class to encapsulate tracing related settings + */ +public class TelemetrySettings { + public static final Setting TRACER_ENABLED_SETTING = Setting.boolSetting( + "telemetry.tracer.enabled", + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private volatile boolean tracingEnabled; + + public TelemetrySettings(Settings settings, ClusterSettings clusterSettings) { + this.tracingEnabled = TRACER_ENABLED_SETTING.get(settings); + + clusterSettings.addSettingsUpdateConsumer(TRACER_ENABLED_SETTING, this::setTracingEnabled); + } + + public void setTracingEnabled(boolean tracingEnabled) { + this.tracingEnabled = tracingEnabled; + } + + public boolean isTracingEnabled() { + return tracingEnabled; + } + +} diff --git a/server/src/main/java/org/opensearch/telemetry/package-info.java b/server/src/main/java/org/opensearch/telemetry/package-info.java new file mode 100644 index 0000000000000..4545f0ef5990e --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for telemetry. + */ +package org.opensearch.telemetry; diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java b/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java new file mode 100644 index 0000000000000..3d7f8133788ce --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.telemetry.tracing.noop.NoopTracer; + +import java.util.Optional; + +/** + * No-op implementation of TracerFactory + */ +public class NoopTracerFactory extends TracerFactory { + public NoopTracerFactory() { + super(null, Optional.empty(), null); + } + + @Override + public Tracer getTracer() { + return NoopTracer.INSTANCE; + } + + @Override + public void close() { + + } + +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java b/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java new file mode 100644 index 0000000000000..0d0b795fdc715 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.util.concurrent.ThreadContextStatePropagator; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Core's ThreadContext based TracerContextStorage implementation + */ +public class ThreadContextBasedTracerContextStorage implements TracerContextStorage, ThreadContextStatePropagator { + + private final ThreadContext threadContext; + + private final TracingTelemetry tracingTelemetry; + + public ThreadContextBasedTracerContextStorage(ThreadContext threadContext, TracingTelemetry tracingTelemetry) { + this.threadContext = Objects.requireNonNull(threadContext); + this.tracingTelemetry = Objects.requireNonNull(tracingTelemetry); + this.threadContext.registerThreadContextStatePropagator(this); + } + + @Override + public Span get(String key) { + return getCurrentSpan(key); + } + + @Override + public void put(String key, Span span) { + if (span == null) { + return; + } + SpanReference currentSpanRef = threadContext.getTransient(key); + if (currentSpanRef == null) { + threadContext.putTransient(key, new SpanReference(span)); + } else { + currentSpanRef.setSpan(span); + } + } + + @Override + public Map transients(Map source) { + final Map transients = new HashMap<>(); + + if (source.containsKey(CURRENT_SPAN)) { + final SpanReference current = (SpanReference) source.get(CURRENT_SPAN); + if (current != null) { + transients.put(CURRENT_SPAN, new SpanReference(current.getSpan())); + } + } + + return transients; + } + + @Override + public Map headers(Map source) { + final Map headers = new HashMap<>(); + + if (source.containsKey(CURRENT_SPAN)) { + final SpanReference current = (SpanReference) source.get(CURRENT_SPAN); + if (current != null) { + tracingTelemetry.getContextPropagator().inject(current.getSpan(), (key, value) -> headers.put(key, value)); + } + } + + return headers; + } + + Span getCurrentSpan(String key) { + Optional optionalSpanFromContext = spanFromThreadContext(key); + return optionalSpanFromContext.orElse(spanFromHeader()); + } + + private Optional spanFromThreadContext(String key) { + SpanReference currentSpanRef = threadContext.getTransient(key); + return (currentSpanRef == null) ? Optional.empty() : Optional.ofNullable(currentSpanRef.getSpan()); + } + + private Span spanFromHeader() { + return tracingTelemetry.getContextPropagator().extract(threadContext.getHeaders()); + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java b/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java new file mode 100644 index 0000000000000..8228cded4c822 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Optional; + +/** + * TracerManager represents a single global class that is used to access tracers. + * + * The Tracer singleton object can be retrieved using tracerManager.getTracer(). The TracerManager object + * is created during class initialization and cannot subsequently be changed. + */ +public class TracerFactory implements Closeable { + + private static final Logger logger = LogManager.getLogger(TracerFactory.class); + + private final TelemetrySettings telemetrySettings; + private final Tracer defaultTracer; + + public TracerFactory(TelemetrySettings telemetrySettings, Optional telemetry, ThreadContext threadContext) { + this.telemetrySettings = telemetrySettings; + this.defaultTracer = telemetry.map(Telemetry::getTracingTelemetry) + .map(tracingTelemetry -> createDefaultTracer(tracingTelemetry, threadContext)) + .orElse(NoopTracer.INSTANCE); + } + + /** + * Returns the tracer instance + * @return tracer instance + */ + public Tracer getTracer() { + return telemetrySettings.isTracingEnabled() ? defaultTracer : NoopTracer.INSTANCE; + } + + /** + * Closes the {@link Tracer} + */ + @Override + public void close() { + try { + defaultTracer.close(); + } catch (IOException e) { + logger.warn("Error closing tracer", e); + } + } + + private Tracer createDefaultTracer(TracingTelemetry tracingTelemetry, ThreadContext threadContext) { + TracerContextStorage tracerContextStorage = new ThreadContextBasedTracerContextStorage( + threadContext, + tracingTelemetry + ); + return new DefaultTracer(tracingTelemetry, tracerContextStorage); + } + +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/package-info.java b/server/src/main/java/org/opensearch/telemetry/tracing/package-info.java new file mode 100644 index 0000000000000..4ac1e4c212c81 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.tracing; diff --git a/server/src/test/java/org/opensearch/telemetry/TelemetryModuleTests.java b/server/src/test/java/org/opensearch/telemetry/TelemetryModuleTests.java new file mode 100644 index 0000000000000..45344ab4253f7 --- /dev/null +++ b/server/src/test/java/org/opensearch/telemetry/TelemetryModuleTests.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import org.opensearch.plugins.TelemetryPlugin; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TelemetryModuleTests extends OpenSearchTestCase { + + public void testGetTelemetrySuccess() { + TelemetryPlugin telemetryPlugin = mock(TelemetryPlugin.class); + when(telemetryPlugin.getName()).thenReturn("otel"); + Telemetry mockTelemetry = mock(Telemetry.class); + when(telemetryPlugin.getTelemetry(any())).thenReturn(Optional.of(mockTelemetry)); + List telemetryPlugins = List.of(telemetryPlugin); + + TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, any()); + + assertTrue(telemetryModule.getTelemetry().isPresent()); + assertEquals(mockTelemetry, telemetryModule.getTelemetry().get()); + } + + public void testGetTelemetryWithMultipleInstalledPlugins() { + TelemetryPlugin telemetryPlugin1 = mock(TelemetryPlugin.class); + TelemetryPlugin telemetryPlugin2 = mock(TelemetryPlugin.class); + when(telemetryPlugin1.getName()).thenReturn("otel"); + Telemetry mockTelemetry1 = mock(Telemetry.class); + Telemetry mockTelemetry2 = mock(Telemetry.class); + + when(telemetryPlugin1.getTelemetry(any())).thenReturn(Optional.of(mockTelemetry1)); + when(telemetryPlugin2.getTelemetry(any())).thenReturn(Optional.of(mockTelemetry2)); + + List telemetryPlugins = List.of(telemetryPlugin1, telemetryPlugin2); + + try { + TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, any()); + } catch (Exception e) { + assertEquals("Cannot register more than one telemetry", e.getMessage()); + } + + } + + public void testGetTelemetryWithNoPlugins() { + TelemetryPlugin telemetryPlugin = mock(TelemetryPlugin.class); + when(telemetryPlugin.getName()).thenReturn("otel"); + TelemetryModule telemetryModule = new TelemetryModule(List.of(telemetryPlugin), any()); + + assertFalse(telemetryModule.getTelemetry().isPresent()); + + } + +} diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java new file mode 100644 index 0000000000000..7968c6c43afb4 --- /dev/null +++ b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.junit.After; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.telemetry.tracing.noop.NoopTracer; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TracerFactoryTests extends OpenSearchTestCase { + + private TracerFactory tracerFactory; + + @After + public void close() { + tracerFactory.close(); + } + + public void testGetTracerWithTracingDisabledReturnsNoopTracer() { + Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), false).build(); + TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); + Telemetry mockTelemetry = mock(Telemetry.class); + when(mockTelemetry.getTracingTelemetry()).thenReturn(mock(TracingTelemetry.class)); + tracerFactory = new TracerFactory(telemetrySettings, Optional.of(mockTelemetry), new ThreadContext(Settings.EMPTY)); + + Tracer tracer = tracerFactory.getTracer(); + assertTrue(tracer instanceof NoopTracer); + assertTrue(tracer.startSpan("foo") == Scope.NO_OP); + } + + public void testGetTracerWithTracingEnabledReturnsDefaultTracer() { + Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true).build(); + TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); + Telemetry mockTelemetry = mock(Telemetry.class); + when(mockTelemetry.getTracingTelemetry()).thenReturn(mock(TracingTelemetry.class)); + tracerFactory = new TracerFactory(telemetrySettings, Optional.of(mockTelemetry), new ThreadContext(Settings.EMPTY)); + + Tracer tracer = tracerFactory.getTracer(); + assertTrue(tracer instanceof DefaultTracer); + + } + + private Set> getClusterSettings() { + Set> allTracerSettings = new HashSet<>(); + ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add)); + return allTracerSettings; + } +} From 3a1dd3ac2acfc5f989f43276eb1612ac3d354ac2 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Wed, 28 Jun 2023 09:09:12 -0700 Subject: [PATCH 015/186] Added release notes for 1.3.11 (#8302) Signed-off-by: Owais Kazi --- release-notes/opensearch.release-notes-1.3.11.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 release-notes/opensearch.release-notes-1.3.11.md diff --git a/release-notes/opensearch.release-notes-1.3.11.md b/release-notes/opensearch.release-notes-1.3.11.md new file mode 100644 index 0000000000000..b2589d94b4f57 --- /dev/null +++ b/release-notes/opensearch.release-notes-1.3.11.md @@ -0,0 +1,13 @@ +## 2023-06-27 Version 1.3.11 Release Notes + +### Upgrades +- Upgrade `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901)) +- Upgrade `spock-core` from 1.3-groovy-2.5 to 2.3-groovy-2.5 ([#8119](https://github.com/opensearch-project/OpenSearch/pull/8119)) +- Upgrade `com.google.guava:guava` from 31.0.1-jre to 32.0.1-jre ([#8107](https://github.com/opensearch-project/OpenSearch/pull/8107)) +- Upgrade versions of gradle-info-plugin and nebula-publishing-plugin ([#8150](https://github.com/opensearch-project/OpenSearch/pull/8150)) +- Upgrade `json-smart` from 2.4.7 to 2.4.11 in plugins/repository-hdfs/ ([#8160](https://github.com/opensearch-project/OpenSearch/pull/8160)) +- Upgrade `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) + +### Bug Fixes +- Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591)) +- Force nebula plugins to use latest org.bouncycastle:* artifacts ([#8233](https://github.com/opensearch-project/OpenSearch/pull/8233)) From b594f52f655e3ec5de57cb827b9b02800ce1666b Mon Sep 17 00:00:00 2001 From: Harish Bhakuni Date: Wed, 28 Jun 2023 10:03:17 -0700 Subject: [PATCH 016/186] [Snapshot Interop] Change Version Checks from 3.0 to 2.9 in Restore Snapshot Changes. (#8319) Signed-off-by: Harish Bhakuni Co-authored-by: Harish Bhakuni --- .../cluster/snapshots/restore/RestoreSnapshotRequest.java | 4 ++-- .../java/org/opensearch/cluster/routing/RecoverySource.java | 4 ++-- .../main/java/org/opensearch/snapshots/RestoreService.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 7ff16fcf6fc03..9ab66d726854e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -150,7 +150,7 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_7_0)) { storageType = in.readEnum(StorageType.class); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_9_0)) { sourceRemoteStoreRepository = in.readOptionalString(); } } @@ -174,7 +174,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_7_0)) { out.writeEnum(storageType); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeOptionalString(sourceRemoteStoreRepository); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index 1af4fbe8ffb45..41e06ac2a8b24 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -293,7 +293,7 @@ public SnapshotRecoverySource( } else { isSearchableSnapshot = false; } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_9_0)) { remoteStoreIndexShallowCopy = in.readBoolean(); sourceRemoteStoreRepository = in.readOptionalString(); } else { @@ -345,7 +345,7 @@ protected void writeAdditionalFields(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_7_0)) { out.writeBoolean(isSearchableSnapshot); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeBoolean(remoteStoreIndexShallowCopy); out.writeOptionalString(sourceRemoteStoreRepository); } diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index bd162914e830e..ebd0e59599c21 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -453,7 +453,7 @@ public ClusterState execute(ClusterState currentState) { final boolean isRemoteStoreShallowCopy = Boolean.TRUE.equals( snapshotInfo.isRemoteStoreIndexShallowCopyEnabled() ) && metadata.index(index).getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false); - if (isRemoteStoreShallowCopy && !currentState.getNodes().getMinNodeVersion().onOrAfter(Version.V_3_0_0)) { + if (isRemoteStoreShallowCopy && !currentState.getNodes().getMinNodeVersion().onOrAfter(Version.V_2_9_0)) { throw new SnapshotRestoreException( snapshot, "cannot restore shallow copy snapshot for index [" From 270abeba62a78d4ab0aba58854ae4458d2f6bd90 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 28 Jun 2023 11:47:01 -0700 Subject: [PATCH 017/186] [Segment Replication] Update bwc test to rely on segments for verification (#8267) * [Segment Replication] Use _cat/segments vs index stats + _search to verify doc count Signed-off-by: Suraj Singh Self review Signed-off-by: Suraj Singh remove unused imports Signed-off-by: Suraj Singh Handle 0 doc count segments Signed-off-by: Suraj Singh * Use 1 minute timeout for assertBusy validations and comments Signed-off-by: Suraj Singh --------- Signed-off-by: Suraj Singh --- .../org/opensearch/upgrades/IndexingIT.java | 92 +++++++++---------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index 173aa9f6557d2..93c0bc96a5183 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -48,16 +48,12 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; import static org.opensearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.either; /** * Basic test that indexed documents survive the rolling restart. See @@ -88,52 +84,51 @@ private void printClusterNodes() throws IOException, ParseException, URISyntaxEx } // Verifies that for each shard copy holds same document count across all containing nodes. - private void waitForSearchableDocs(String index, int shardCount) throws Exception { - Map primaryShardToNodeIDMap = new HashMap<>(); - Map replicaShardToNodeIDMap = new HashMap<>(); + private void waitForSearchableDocs(String index, int shardCount, int replicaCount) throws Exception { + assertTrue(shardCount > 0); + assertTrue(replicaCount > 0); waitForClusterHealthWithNoShardMigration(index, "green"); logger.info("--> _cat/shards before search \n{}", EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/shards?v")).getEntity())); + // Verify segment replication stats verifySegmentStats(index); - Request request = new Request("GET", index + "/_stats"); - request.addParameter("level", "shards"); - Response response = client().performRequest(request); - for (int shardNumber = 0; shardNumber < shardCount; shardNumber++) { - List shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards." + shardNumber); - for (Object shard : shardStats) { - final String nodeId = ObjectPath.evaluate(shard, "routing.node"); - final Boolean primary = ObjectPath.evaluate(shard, "routing.primary"); - if (primary) { - primaryShardToNodeIDMap.putIfAbsent(shardNumber, nodeId); - } else { - replicaShardToNodeIDMap.putIfAbsent(shardNumber, nodeId); + + // Verify segment store + assertBusy(() -> { + /** + * Use default tabular output and sort response based on shard,segment,primaryOrReplica columns to allow line by + * line parsing where records related to a segment (e.g. _0) are chunked together with first record belonging + * to primary while remaining *replicaCount* records belongs to replica copies + * */ + Request segrepStatsRequest = new Request("GET", "/_cat/segments/" + index + "?s=shard,segment,primaryOrReplica"); + segrepStatsRequest.addParameter("h", "index,shard,primaryOrReplica,segment,docs.count"); + Response segrepStatsResponse = client().performRequest(segrepStatsRequest); + logger.info("--> _cat/segments response\n {}", EntityUtils.toString(segrepStatsResponse.getEntity())); + List responseList = Streams.readAllLines(segrepStatsResponse.getEntity().getContent()); + for (int segmentsIndex=0; segmentsIndex < responseList.size();) { + String[] primaryRow = responseList.get(segmentsIndex++).split(" +"); + String shardId = primaryRow[0] + primaryRow[1]; + assertTrue(primaryRow[2].equals("p")); + for(int replicaIndex = 1; replicaIndex <= replicaCount; replicaIndex++) { + String[] replicaRow = responseList.get(segmentsIndex).split(" +"); + String replicaShardId = replicaRow[0] + replicaRow[1]; + // When segment has 0 doc count, not all replica copies posses that segment. Skip to next segment + if (replicaRow[2].equals("p")) { + assertTrue(primaryRow[4].equals("0")); + break; + } + // verify same shard id + assertTrue(replicaShardId.equals(shardId)); + // verify replica row + assertTrue(replicaRow[2].equals("r")); + // Verify segment name matches e.g. _0 + assertTrue(replicaRow[3].equals(primaryRow[3])); + // Verify doc count matches + assertTrue(replicaRow[4].equals(primaryRow[4])); + segmentsIndex++; } } - } - logger.info("--> primaryShardToNodeIDMap {}", primaryShardToNodeIDMap); - logger.info("--> replicaShardToNodeIDMap {}", replicaShardToNodeIDMap); - - for (int shardNumber = 0; shardNumber < shardCount; shardNumber++) { - Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); - searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); - searchTestIndexRequest.addParameter("filter_path", "hits.total"); - searchTestIndexRequest.addParameter("preference", "_shards:" + shardNumber + "|_only_nodes:" + primaryShardToNodeIDMap.get(shardNumber)); - Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); - final int primaryHits = ObjectPath.createFromResponse(searchTestIndexResponse).evaluate("hits.total"); - final int shardNum = shardNumber; - // Verify replica shard doc count only when available. - if (replicaShardToNodeIDMap.get(shardNum) != null) { - assertBusy(() -> { - Request replicaRequest = new Request("POST", "/" + index + "/_search"); - replicaRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); - replicaRequest.addParameter("filter_path", "hits.total"); - replicaRequest.addParameter("preference", "_shards:" + shardNum + "|_only_nodes:" + replicaShardToNodeIDMap.get(shardNum)); - Response replicaResponse = client().performRequest(replicaRequest); - int replicaHits = ObjectPath.createFromResponse(replicaResponse).evaluate("hits.total"); - assertEquals("Doc count mismatch for shard " + shardNum + ". Primary hits " + primaryHits + " Replica hits " + replicaHits, primaryHits, replicaHits); - }, 1, TimeUnit.MINUTES); - } - } + }, 1, TimeUnit.MINUTES); } private void waitForClusterHealthWithNoShardMigration(String indexName, String status) throws IOException { @@ -156,7 +151,7 @@ private void verifySegmentStats(String indexName) throws Exception { String[] elements = statLine.split(" +"); assertEquals("Replica shard " + elements[0] + "not upto date with primary ", 0, Integer.parseInt(elements[2])); } - }); + }, 1, TimeUnit.MINUTES); } public void testIndexing() throws IOException, ParseException { @@ -307,7 +302,7 @@ public void testIndexingWithSegRep() throws Exception { throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); } - waitForSearchableDocs(indexName, shardCount); + waitForSearchableDocs(indexName, shardCount, replicaCount); assertCount(indexName, expectedCount); if (CLUSTER_TYPE != ClusterType.OLD) { @@ -318,17 +313,16 @@ public void testIndexingWithSegRep() throws Exception { toBeDeleted.addParameter("refresh", "true"); toBeDeleted.setJsonEntity("{\"f1\": \"delete-me\"}"); client().performRequest(toBeDeleted); - waitForSearchableDocs(indexName, shardCount); + waitForSearchableDocs(indexName, shardCount, replicaCount); assertCount(indexName, expectedCount + 6); logger.info("--> Delete previously added doc and verify doc count"); Request delete = new Request("DELETE", "/" + indexName + "/_doc/to_be_deleted"); delete.addParameter("refresh", "true"); client().performRequest(delete); - waitForSearchableDocs(indexName, shardCount); + waitForSearchableDocs(indexName, shardCount, replicaCount); assertCount(indexName, expectedCount + 5); } - logger.info("--> _cat/shards post execution \n{}", EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/shards?v")).getEntity())); } public void testAutoIdWithOpTypeCreate() throws IOException { From 9aeb15699ef30312715a624af81aa826c4e4c42e Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Wed, 28 Jun 2023 14:57:12 -0500 Subject: [PATCH 018/186] [Mute] SegRepUsingRemoteStoreIT.testPressureServiceStats (#8324) Mutes repeated failing test. Signed-off-by: Nicholas Walter Knize --- .../remotestore/SegmentReplicationUsingRemoteStoreIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index ad6e358cb9da1..01fb91f83aa02 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -63,4 +63,10 @@ public void setup() { public void teardown() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7592") + @Override + public void testPressureServiceStats() throws Exception { + super.testPressureServiceStats(); + } } From 9856cb77bc1bf6908210d91af7f95cc29d41b3a5 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Wed, 28 Jun 2023 23:12:09 +0200 Subject: [PATCH 019/186] Bump BouncyCastle from jdk15on to jdk15to18 (#8247) jdk15on are not supported anymore since based development was moved from jsk15on to jdk18on. jdk15to18 contains fixed for: - CVE-2023-33201 - CVE-2022-45146 Signed-off-by: Andrey Pleskach --- CHANGELOG.md | 5 +++++ buildSrc/version.properties | 2 +- plugins/identity-shiro/build.gradle | 2 +- .../identity-shiro/licenses/bcprov-jdk15on-1.70.jar.sha1 | 1 - .../identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 | 1 + ...rov-jdk15on-LICENSE.txt => bcprov-jdk15to18-LICENSE.txt} | 0 ...cprov-jdk15on-NOTICE.txt => bcprov-jdk15to18-NOTICE.txt} | 0 plugins/ingest-attachment/build.gradle | 6 +++--- .../ingest-attachment/licenses/bcmail-jdk15on-1.70.jar.sha1 | 1 - .../licenses/bcmail-jdk15to18-1.75.jar.sha1 | 1 + ...ail-jdk15on-LICENSE.txt => bcmail-jdk15to18-LICENSE.txt} | 0 ...cmail-jdk15on-NOTICE.txt => bcmail-jdk15to18-NOTICE.txt} | 0 .../ingest-attachment/licenses/bcpkix-jdk15on-1.70.jar.sha1 | 1 - .../licenses/bcpkix-jdk15to18-1.75.jar.sha1 | 1 + ...kix-jdk15on-LICENSE.txt => bcpkix-jdk15to18-LICENSE.txt} | 0 ...cpkix-jdk15on-NOTICE.txt => bcpkix-jdk15to18-NOTICE.txt} | 0 .../ingest-attachment/licenses/bcprov-jdk15on-1.70.jar.sha1 | 1 - .../licenses/bcprov-jdk15to18-1.75.jar.sha1 | 1 + ...rov-jdk15on-LICENSE.txt => bcprov-jdk15to18-LICENSE.txt} | 0 ...cprov-jdk15on-NOTICE.txt => bcprov-jdk15to18-NOTICE.txt} | 0 .../src/main/plugin-metadata/plugin-security.policy | 3 --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 22 files changed, 15 insertions(+), 13 deletions(-) delete mode 100644 plugins/identity-shiro/licenses/bcprov-jdk15on-1.70.jar.sha1 create mode 100644 plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 rename plugins/identity-shiro/licenses/{bcprov-jdk15on-LICENSE.txt => bcprov-jdk15to18-LICENSE.txt} (100%) rename plugins/identity-shiro/licenses/{bcprov-jdk15on-NOTICE.txt => bcprov-jdk15to18-NOTICE.txt} (100%) delete mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15on-1.70.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 rename plugins/ingest-attachment/licenses/{bcmail-jdk15on-LICENSE.txt => bcmail-jdk15to18-LICENSE.txt} (100%) rename plugins/ingest-attachment/licenses/{bcmail-jdk15on-NOTICE.txt => bcmail-jdk15to18-NOTICE.txt} (100%) delete mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.70.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 rename plugins/ingest-attachment/licenses/{bcpkix-jdk15on-LICENSE.txt => bcpkix-jdk15to18-LICENSE.txt} (100%) rename plugins/ingest-attachment/licenses/{bcpkix-jdk15on-NOTICE.txt => bcpkix-jdk15to18-NOTICE.txt} (100%) delete mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15on-1.70.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 rename plugins/ingest-attachment/licenses/{bcprov-jdk15on-LICENSE.txt => bcprov-jdk15to18-LICENSE.txt} (100%) rename plugins/ingest-attachment/licenses/{bcprov-jdk15on-NOTICE.txt => bcprov-jdk15to18-NOTICE.txt} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c370dbe7b0d9..7f4a6f38b578e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344) - Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928))) - Add Opentelemetry dependencies ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) +- Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) + + ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 735a9fe4f8255..f9eac9516cb18 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -48,7 +48,7 @@ reactivestreams = 1.0.4 # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) # - distribution/tools/plugin-cli -bouncycastle=1.70 +bouncycastle=1.75 # test dependencies randomizedrunner = 2.7.1 junit = 4.13.2 diff --git a/plugins/identity-shiro/build.gradle b/plugins/identity-shiro/build.gradle index 22dc21864b620..baa3464d0a98e 100644 --- a/plugins/identity-shiro/build.gradle +++ b/plugins/identity-shiro/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation 'org.passay:passay:1.6.3' - implementation "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}" + implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" testImplementation project(path: ':modules:transport-netty4') // for http testImplementation project(path: ':plugins:transport-nio') // for http diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15on-1.70.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk15on-1.70.jar.sha1 deleted file mode 100644 index f5e89c0f5ed45..0000000000000 --- a/plugins/identity-shiro/licenses/bcprov-jdk15on-1.70.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4636a0d01f74acaf28082fb62b317f1080118371 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 new file mode 100644 index 0000000000000..9911bb75f9209 --- /dev/null +++ b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 @@ -0,0 +1 @@ +df22e1b6a9f6b218913f5b68dd16641344397fe0 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15on-LICENSE.txt b/plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt similarity index 100% rename from plugins/identity-shiro/licenses/bcprov-jdk15on-LICENSE.txt rename to plugins/identity-shiro/licenses/bcprov-jdk15to18-LICENSE.txt diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15on-NOTICE.txt b/plugins/identity-shiro/licenses/bcprov-jdk15to18-NOTICE.txt similarity index 100% rename from plugins/identity-shiro/licenses/bcprov-jdk15on-NOTICE.txt rename to plugins/identity-shiro/licenses/bcprov-jdk15to18-NOTICE.txt diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 1f3c80909733c..62651216c8144 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -71,9 +71,9 @@ dependencies { api "org.apache.pdfbox:fontbox:${versions.pdfbox}" api "org.apache.pdfbox:jempbox:1.8.17" api "commons-logging:commons-logging:${versions.commonslogging}" - api "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}" - api "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}" - api "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}" + api "org.bouncycastle:bcmail-jdk15to18:${versions.bouncycastle}" + api "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" + api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" // OpenOffice api "org.apache.poi:poi-ooxml:${versions.poi}" api "org.apache.poi:poi:${versions.poi}" diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.70.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.70.jar.sha1 deleted file mode 100644 index 672e479eda8d7..0000000000000 --- a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.70.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08f4aafad90f6cc7f16b9992279828ae848c9e0d \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 new file mode 100644 index 0000000000000..e6840a9b02b38 --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 @@ -0,0 +1 @@ +b316bcd094e3917b1ece93a6edbab93f8315fb3b \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-LICENSE.txt b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-LICENSE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcmail-jdk15on-LICENSE.txt rename to plugins/ingest-attachment/licenses/bcmail-jdk15to18-LICENSE.txt diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-NOTICE.txt b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcmail-jdk15on-NOTICE.txt rename to plugins/ingest-attachment/licenses/bcmail-jdk15to18-NOTICE.txt diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.70.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.70.jar.sha1 deleted file mode 100644 index e348463a21257..0000000000000 --- a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.70.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f81e5af49571a9d5a109a88f239a73ce87055417 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 new file mode 100644 index 0000000000000..9181b1c3ab1b6 --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 @@ -0,0 +1 @@ +f16e5252ad7a46d5eaf255231b0a5da307599082 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-LICENSE.txt b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-LICENSE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcpkix-jdk15on-LICENSE.txt rename to plugins/ingest-attachment/licenses/bcpkix-jdk15to18-LICENSE.txt diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-NOTICE.txt b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcpkix-jdk15on-NOTICE.txt rename to plugins/ingest-attachment/licenses/bcpkix-jdk15to18-NOTICE.txt diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.70.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.70.jar.sha1 deleted file mode 100644 index f5e89c0f5ed45..0000000000000 --- a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.70.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4636a0d01f74acaf28082fb62b317f1080118371 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 new file mode 100644 index 0000000000000..9911bb75f9209 --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 @@ -0,0 +1 @@ +df22e1b6a9f6b218913f5b68dd16641344397fe0 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-LICENSE.txt b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcprov-jdk15on-LICENSE.txt rename to plugins/ingest-attachment/licenses/bcprov-jdk15to18-LICENSE.txt diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-NOTICE.txt b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/bcprov-jdk15on-NOTICE.txt rename to plugins/ingest-attachment/licenses/bcprov-jdk15to18-NOTICE.txt diff --git a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy index 0fa85f6f040f6..4b90f9a21aae4 100644 --- a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy @@ -35,9 +35,6 @@ grant { // needed to apply additional sandboxing to tika parsing permission java.security.SecurityPermission "createAccessControlContext"; - // TODO: fix PDFBox not to actually install bouncy castle like this - permission java.security.SecurityPermission "putProviderProperty.BC"; - permission java.security.SecurityPermission "insertProvider"; // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597 permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; // needed by xmlbeans, as part of POI for MS xml docs diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index de6f69a4fd4ce..310b088674d5c 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -51,7 +51,7 @@ dependencies { api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "io.netty:netty-all:${versions.netty}" api 'com.google.code.gson:gson:2.10.1' - api "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}" + api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" From 46c9a211b6b9490f6a7ac9425e946986cd51bed2 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Wed, 28 Jun 2023 14:26:53 -0700 Subject: [PATCH 020/186] [Search Pipelines] Add stats for search pipelines (#8053) * [Search Pipelines] Add stats for search pipelines This adds statistics on executions and time spent on search pipeline operations, similar to the stats that are available for ingest pipelines. Signed-off-by: Michael Froh * Compare parsed JSON structure, not exact JSON string As @lukas-vlcek pointed out, asserting equality with an exact JSON string is sensitive to formatting, which makes the test brittle. Instead, we can parse the expected JSON and compare as Maps. Signed-off-by: Michael Froh * Refactor to common stats/metrics classes Search pipelines and ingest pipelines had identical functionality for tracking metrics around operations and converting those to immutable "stats" objects. That approach isn't even really specific to pipelines, but can be used to track metrics on any repeated operation, so I moved that common logic to the common.metrics package. Signed-off-by: Michael Froh * Split pipeline metrics tracking into its own class Thanks @saratvemulapalli for the suggestion! This lets the Pipeline class focus on transforming requests / responses, while the subclass focuses on tracking and managing metrics. Signed-off-by: Michael Froh --------- Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../ingest/common/IngestRestartIT.java | 2 +- .../admin/cluster/node/stats/NodeStats.java | 24 +- .../cluster/node/stats/NodesStatsRequest.java | 3 +- .../node/stats/TransportNodesStatsAction.java | 3 +- .../cluster/stats/ClusterStatsNodes.java | 19 +- .../stats/TransportClusterStatsAction.java | 1 + .../opensearch/common/metrics/MeanMetric.java | 5 + .../common/metrics/OperationMetrics.java | 68 ++++ .../common/metrics/OperationStats.java | 107 +++++ .../opensearch/ingest/CompoundProcessor.java | 17 +- .../ingest/ConditionalProcessor.java | 13 +- .../org/opensearch/ingest/IngestMetric.java | 112 ------ .../org/opensearch/ingest/IngestService.java | 39 +- .../org/opensearch/ingest/IngestStats.java | 131 +------ .../java/org/opensearch/ingest/Pipeline.java | 13 +- .../java/org/opensearch/node/NodeService.java | 6 +- .../opensearch/search/pipeline/Pipeline.java | 170 ++++---- .../search/pipeline/PipelineWithMetrics.java | 227 +++++++++++ .../pipeline/SearchPipelineService.java | 52 ++- .../search/pipeline/SearchPipelineStats.java | 367 ++++++++++++++++++ .../cluster/node/stats/NodeStatsTests.java | 49 +-- .../cluster/stats/ClusterStatsNodesTests.java | 16 +- .../opensearch/cluster/DiskUsageTests.java | 6 + .../ingest/CompoundProcessorTests.java | 11 +- .../ingest/ConditionalProcessorTests.java | 11 +- .../opensearch/ingest/IngestServiceTests.java | 13 +- .../opensearch/ingest/IngestStatsTests.java | 27 +- .../ingest/PipelineProcessorTests.java | 31 +- .../pipeline/SearchPipelineServiceTests.java | 123 ++++++ .../pipeline/SearchPipelineStatsTests.java | 185 +++++++++ .../MockInternalClusterInfoService.java | 3 +- .../opensearch/test/InternalTestCluster.java | 1 + 33 files changed, 1398 insertions(+), 458 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java create mode 100644 server/src/main/java/org/opensearch/common/metrics/OperationStats.java delete mode 100644 server/src/main/java/org/opensearch/ingest/IngestMetric.java create mode 100644 server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java create mode 100644 server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java create mode 100644 server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f4a6f38b578e..109401a46e5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement concurrent aggregations support without profile option ([#7514](https://github.com/opensearch-project/OpenSearch/pull/7514)) - Add dynamic index and cluster setting for concurrent segment search ([#7956](https://github.com/opensearch-project/OpenSearch/pull/7956)) - Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967)) +- [Search pipelines] Added search pipelines output to node stats ([#8053](https://github.com/opensearch-project/OpenSearch/pull/8053)) - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020)) - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038)) - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) diff --git a/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java index 784dad8cea49f..1f0680b27796d 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java @@ -132,7 +132,7 @@ public void testFailureInConditionalProcessor() { for (int k = 0; k < nodeCount; k++) { List stats = r.getNodes().get(k).getIngestStats().getProcessorStats().get(pipelineId); for (IngestStats.ProcessorStat st : stats) { - assertThat(st.getStats().getIngestCurrent(), greaterThanOrEqualTo(0L)); + assertThat(st.getStats().getCurrent(), greaterThanOrEqualTo(0L)); } } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index c183562e2e85a..6b8e06594acb7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -59,6 +59,7 @@ import org.opensearch.script.ScriptCacheStats; import org.opensearch.script.ScriptStats; import org.opensearch.search.backpressure.stats.SearchBackpressureStats; +import org.opensearch.search.pipeline.SearchPipelineStats; import org.opensearch.tasks.TaskCancellationStats; import org.opensearch.threadpool.ThreadPoolStats; import org.opensearch.transport.TransportStats; @@ -138,6 +139,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private TaskCancellationStats taskCancellationStats; + @Nullable + private SearchPipelineStats searchPipelineStats; + public NodeStats(StreamInput in) throws IOException { super(in); timestamp = in.readVLong(); @@ -189,6 +193,11 @@ public NodeStats(StreamInput in) throws IOException { } else { taskCancellationStats = null; } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { // TODO Update to 2_9_0 when we backport to 2.x + searchPipelineStats = in.readOptionalWriteable(SearchPipelineStats::new); + } else { + searchPipelineStats = null; + } } public NodeStats( @@ -214,7 +223,8 @@ public NodeStats( @Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats, @Nullable WeightedRoutingStats weightedRoutingStats, @Nullable FileCacheStats fileCacheStats, - @Nullable TaskCancellationStats taskCancellationStats + @Nullable TaskCancellationStats taskCancellationStats, + @Nullable SearchPipelineStats searchPipelineStats ) { super(node); this.timestamp = timestamp; @@ -239,6 +249,7 @@ public NodeStats( this.weightedRoutingStats = weightedRoutingStats; this.fileCacheStats = fileCacheStats; this.taskCancellationStats = taskCancellationStats; + this.searchPipelineStats = searchPipelineStats; } public long getTimestamp() { @@ -371,6 +382,11 @@ public TaskCancellationStats getTaskCancellationStats() { return taskCancellationStats; } + @Nullable + public SearchPipelineStats getSearchPipelineStats() { + return searchPipelineStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -411,6 +427,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeOptionalWriteable(taskCancellationStats); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { // TODO: Update to 2_9_0 once we backport to 2.x + out.writeOptionalWriteable(searchPipelineStats); + } } @Override @@ -498,6 +517,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (getTaskCancellationStats() != null) { getTaskCancellationStats().toXContent(builder, params); } + if (getSearchPipelineStats() != null) { + getSearchPipelineStats().toXContent(builder, params); + } return builder; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 68f391b91507c..f37a837c6f0ef 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -211,7 +211,8 @@ public enum Metric { CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling"), WEIGHTED_ROUTING_STATS("weighted_routing"), FILE_CACHE_STATS("file_cache"), - TASK_CANCELLATION("task_cancellation"); + TASK_CANCELLATION("task_cancellation"), + SEARCH_PIPELINE("search_pipeline"); private String metricName; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 6aadf546d30f7..660142f05bab2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -123,7 +123,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) { NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics), NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics), NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics), - NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics) + NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics), + NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics) ); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 828f3a2e5e842..25a915833c7e2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -40,6 +40,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; @@ -800,18 +801,18 @@ static class IngestStats implements ToXContentFragment { pipelineIds.add(processorStats.getKey()); for (org.opensearch.ingest.IngestStats.ProcessorStat stat : processorStats.getValue()) { stats.compute(stat.getType(), (k, v) -> { - org.opensearch.ingest.IngestStats.Stats nodeIngestStats = stat.getStats(); + OperationStats nodeIngestStats = stat.getStats(); if (v == null) { return new long[] { - nodeIngestStats.getIngestCount(), - nodeIngestStats.getIngestFailedCount(), - nodeIngestStats.getIngestCurrent(), - nodeIngestStats.getIngestTimeInMillis() }; + nodeIngestStats.getCount(), + nodeIngestStats.getFailedCount(), + nodeIngestStats.getCurrent(), + nodeIngestStats.getTotalTimeInMillis() }; } else { - v[0] += nodeIngestStats.getIngestCount(); - v[1] += nodeIngestStats.getIngestFailedCount(); - v[2] += nodeIngestStats.getIngestCurrent(); - v[3] += nodeIngestStats.getIngestTimeInMillis(); + v[0] += nodeIngestStats.getCount(); + v[1] += nodeIngestStats.getFailedCount(); + v[2] += nodeIngestStats.getCurrent(); + v[3] += nodeIngestStats.getTotalTimeInMillis(); return v; } }); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 726f8a0de19ae..aee6dfddd203e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -167,6 +167,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false, false, + false, false ); List shardsStats = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java b/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java index 79c04d431e97b..33f12c8cb42d3 100644 --- a/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java +++ b/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java @@ -49,6 +49,11 @@ public void inc(long n) { sum.add(n); } + public void add(MeanMetric other) { + counter.add(other.counter.sum()); + sum.add(other.sum.sum()); + } + public void dec(long n) { counter.decrement(); sum.add(-n); diff --git a/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java b/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java new file mode 100644 index 0000000000000..97fbbc2ce5cde --- /dev/null +++ b/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.metrics; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Mutable tracker of a repeated operation. + * + * @opensearch.internal + */ +public class OperationMetrics { + /** + * The mean time it takes to complete the measured item. + */ + private final MeanMetric time = new MeanMetric(); + /** + * The current count of things being measured. + * Useful when aggregating multiple metrics to see how many things are in flight. + */ + private final AtomicLong current = new AtomicLong(); + /** + * The non-decreasing count of failures + */ + private final CounterMetric failed = new CounterMetric(); + + /** + * Invoked before the given operation begins. + */ + public void before() { + current.incrementAndGet(); + } + + /** + * Invoked upon completion (success or failure) of the given operation + * @param currentTime elapsed time of the operation + */ + public void after(long currentTime) { + current.decrementAndGet(); + time.inc(currentTime); + } + + /** + * Invoked upon failure of the operation. + */ + public void failed() { + failed.inc(); + } + + public void add(OperationMetrics other) { + // Don't try copying over current, since in-flight requests will be linked to the existing metrics instance. + failed.inc(other.failed.count()); + time.add(other.time); + } + + /** + * @return an immutable snapshot of the current metric values. + */ + public OperationStats createStats() { + return new OperationStats(time.count(), time.sum(), current.get(), failed.count()); + } +} diff --git a/server/src/main/java/org/opensearch/common/metrics/OperationStats.java b/server/src/main/java/org/opensearch/common/metrics/OperationStats.java new file mode 100644 index 0000000000000..a820f848393bb --- /dev/null +++ b/server/src/main/java/org/opensearch/common/metrics/OperationStats.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.metrics; + +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** + * An immutable representation of a {@link OperationMetrics} + */ +public class OperationStats implements Writeable, ToXContentFragment { + private final long count; + private final long totalTimeInMillis; + private final long current; + private final long failedCount; + + public OperationStats(long count, long totalTimeInMillis, long current, long failedCount) { + this.count = count; + this.totalTimeInMillis = totalTimeInMillis; + this.current = current; + this.failedCount = failedCount; + } + + /** + * Read from a stream. + */ + public OperationStats(StreamInput in) throws IOException { + count = in.readVLong(); + totalTimeInMillis = in.readVLong(); + current = in.readVLong(); + failedCount = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(count); + out.writeVLong(totalTimeInMillis); + out.writeVLong(current); + out.writeVLong(failedCount); + } + + /** + * @return The total number of executed operations. + */ + public long getCount() { + return count; + } + + /** + * @return The total time spent of in millis. + */ + public long getTotalTimeInMillis() { + return totalTimeInMillis; + } + + /** + * @return The total number of operations currently executing. + */ + public long getCurrent() { + return current; + } + + /** + * @return The total number of operations that have failed. + */ + public long getFailedCount() { + return failedCount; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field("count", count) + .humanReadableField("time_in_millis", "time", new TimeValue(totalTimeInMillis, TimeUnit.MILLISECONDS)) + .field("current", current) + .field("failed", failedCount); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OperationStats that = (OperationStats) o; + return Objects.equals(count, that.count) + && Objects.equals(totalTimeInMillis, that.totalTimeInMillis) + && Objects.equals(failedCount, that.failedCount) + && Objects.equals(current, that.current); + } + + @Override + public int hashCode() { + return Objects.hash(count, totalTimeInMillis, failedCount, current); + } +} diff --git a/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java b/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java index 8cdbc487dc137..a5f4870029e87 100644 --- a/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java @@ -34,6 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.metrics.OperationMetrics; import java.util.ArrayList; import java.util.Arrays; @@ -60,7 +61,7 @@ public class CompoundProcessor implements Processor { private final boolean ignoreFailure; private final List processors; private final List onFailureProcessors; - private final List> processorsWithMetrics; + private final List> processorsWithMetrics; private final LongSupplier relativeTimeProvider; CompoundProcessor(LongSupplier relativeTimeProvider, Processor... processor) { @@ -87,10 +88,10 @@ public CompoundProcessor(boolean ignoreFailure, List processors, List this.onFailureProcessors = onFailureProcessors; this.relativeTimeProvider = relativeTimeProvider; this.processorsWithMetrics = new ArrayList<>(processors.size()); - processors.forEach(p -> processorsWithMetrics.add(new Tuple<>(p, new IngestMetric()))); + processors.forEach(p -> processorsWithMetrics.add(new Tuple<>(p, new OperationMetrics()))); } - List> getProcessorsWithMetrics() { + List> getProcessorsWithMetrics() { return processorsWithMetrics; } @@ -155,17 +156,17 @@ void innerExecute(int currentProcessor, IngestDocument ingestDocument, BiConsume return; } - Tuple processorWithMetric = processorsWithMetrics.get(currentProcessor); + Tuple processorWithMetric = processorsWithMetrics.get(currentProcessor); final Processor processor = processorWithMetric.v1(); - final IngestMetric metric = processorWithMetric.v2(); + final OperationMetrics metric = processorWithMetric.v2(); final long startTimeInNanos = relativeTimeProvider.getAsLong(); - metric.preIngest(); + metric.before(); processor.execute(ingestDocument, (result, e) -> { long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - startTimeInNanos); - metric.postIngest(ingestTimeInMillis); + metric.after(ingestTimeInMillis); if (e != null) { - metric.ingestFailed(); + metric.failed(); if (ignoreFailure) { innerExecute(currentProcessor + 1, ingestDocument, handler); } else { diff --git a/server/src/main/java/org/opensearch/ingest/ConditionalProcessor.java b/server/src/main/java/org/opensearch/ingest/ConditionalProcessor.java index 591a71fd72b8f..8bf489805f7ca 100644 --- a/server/src/main/java/org/opensearch/ingest/ConditionalProcessor.java +++ b/server/src/main/java/org/opensearch/ingest/ConditionalProcessor.java @@ -32,6 +32,7 @@ package org.opensearch.ingest; +import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.script.IngestConditionalScript; import org.opensearch.script.Script; import org.opensearch.script.ScriptException; @@ -66,7 +67,7 @@ public class ConditionalProcessor extends AbstractProcessor implements WrappingP private final Script condition; private final ScriptService scriptService; private final Processor processor; - private final IngestMetric metric; + private final OperationMetrics metric; private final LongSupplier relativeTimeProvider; private final IngestConditionalScript precompiledConditionScript; @@ -86,7 +87,7 @@ public class ConditionalProcessor extends AbstractProcessor implements WrappingP this.condition = script; this.scriptService = scriptService; this.processor = processor; - this.metric = new IngestMetric(); + this.metric = new OperationMetrics(); this.relativeTimeProvider = relativeTimeProvider; try { @@ -114,12 +115,12 @@ public void execute(IngestDocument ingestDocument, BiConsumer { long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - startTimeInNanos); - metric.postIngest(ingestTimeInMillis); + metric.after(ingestTimeInMillis); if (e != null) { - metric.ingestFailed(); + metric.failed(); handler.accept(null, e); } else { handler.accept(result, null); @@ -148,7 +149,7 @@ public Processor getInnerProcessor() { return processor; } - IngestMetric getMetric() { + OperationMetrics getMetric() { return metric; } diff --git a/server/src/main/java/org/opensearch/ingest/IngestMetric.java b/server/src/main/java/org/opensearch/ingest/IngestMetric.java deleted file mode 100644 index 2d4a1dc9cfdee..0000000000000 --- a/server/src/main/java/org/opensearch/ingest/IngestMetric.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ingest; - -import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.common.metrics.MeanMetric; - -import java.util.concurrent.atomic.AtomicLong; - -/** - *

Metrics to measure ingest actions. - *

This counts measure documents and timings for a given scope. - * The scope is determined by the calling code. For example you can use this class to count all documents across all pipeline, - * or you can use this class to count documents for a given pipeline or a specific processor. - * This class does not make assumptions about it's given scope. - * - * @opensearch.internal - */ -class IngestMetric { - - /** - * The time it takes to complete the measured item. - */ - private final MeanMetric ingestTime = new MeanMetric(); - /** - * The current count of things being measure. Should most likely ever be 0 or 1. - * Useful when aggregating multiple metrics to see how many things are in flight. - */ - private final AtomicLong ingestCurrent = new AtomicLong(); - /** - * The ever increasing count of things being measured - */ - private final CounterMetric ingestCount = new CounterMetric(); - /** - * The only increasing count of failures - */ - private final CounterMetric ingestFailed = new CounterMetric(); - - /** - * Call this prior to the ingest action. - */ - void preIngest() { - ingestCurrent.incrementAndGet(); - } - - /** - * Call this after the performing the ingest action, even if the action failed. - * @param ingestTimeInMillis The time it took to perform the action. - */ - void postIngest(long ingestTimeInMillis) { - ingestCurrent.decrementAndGet(); - ingestTime.inc(ingestTimeInMillis); - ingestCount.inc(); - } - - /** - * Call this if the ingest action failed. - */ - void ingestFailed() { - ingestFailed.inc(); - } - - /** - *

Add two sets of metrics together. - *

Note - this method does not add the current count values. - * The current count value is ephemeral and requires a increase/decrease operation pairs to keep the value correct. - * - * @param metrics The metric to add. - */ - void add(IngestMetric metrics) { - ingestCount.inc(metrics.ingestCount.count()); - ingestTime.inc(metrics.ingestTime.sum()); - ingestFailed.inc(metrics.ingestFailed.count()); - } - - /** - * Creates a serializable representation for these metrics. - */ - IngestStats.Stats createStats() { - return new IngestStats.Stats(ingestCount.count(), ingestTime.sum(), ingestCurrent.get(), ingestFailed.count()); - } -} diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java index b9785d9ec036f..0984046ca3077 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestService.java +++ b/server/src/main/java/org/opensearch/ingest/IngestService.java @@ -60,6 +60,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -114,7 +115,7 @@ public class IngestService implements ClusterStateApplier, ReportingService pipelines = Collections.emptyMap(); private final ThreadPool threadPool; - private final IngestMetric totalMetrics = new IngestMetric(); + private final OperationMetrics totalMetrics = new OperationMetrics(); private final List> ingestClusterStateListeners = new CopyOnWriteArrayList<>(); private final ClusterManagerTaskThrottler.ThrottlingKey putPipelineTaskKey; private final ClusterManagerTaskThrottler.ThrottlingKey deletePipelineTaskKey; @@ -440,17 +441,17 @@ Map pipelines() { * Recursive method to obtain all of the non-failure processors for given compoundProcessor. Since conditionals are implemented as * wrappers to the actual processor, always prefer the actual processor's metric over the conditional processor's metric. * @param compoundProcessor The compound processor to start walking the non-failure processors - * @param processorMetrics The list of {@link Processor} {@link IngestMetric} tuples. + * @param processorMetrics The list of {@link Processor} {@link OperationMetrics} tuples. * @return the processorMetrics for all non-failure processor that belong to the original compoundProcessor */ - private static List> getProcessorMetrics( + private static List> getProcessorMetrics( CompoundProcessor compoundProcessor, - List> processorMetrics + List> processorMetrics ) { // only surface the top level non-failure processors, on-failure processor times will be included in the top level non-failure - for (Tuple processorWithMetric : compoundProcessor.getProcessorsWithMetrics()) { + for (Tuple processorWithMetric : compoundProcessor.getProcessorsWithMetrics()) { Processor processor = processorWithMetric.v1(); - IngestMetric metric = processorWithMetric.v2(); + OperationMetrics metric = processorWithMetric.v2(); if (processor instanceof CompoundProcessor) { getProcessorMetrics((CompoundProcessor) processor, processorMetrics); } else { @@ -614,7 +615,7 @@ private void executePipelines( if (Objects.equals(originalIndex, newIndex) == false) { if (hasFinalPipeline && it.hasNext() == false) { - totalMetrics.ingestFailed(); + totalMetrics.failed(); onFailure.accept( slot, new IllegalStateException("final pipeline [" + pipelineId + "] can't change the target index") @@ -680,11 +681,11 @@ public IngestStats stats() { Pipeline pipeline = holder.pipeline; CompoundProcessor rootProcessor = pipeline.getCompoundProcessor(); statsBuilder.addPipelineMetrics(id, pipeline.getMetrics()); - List> processorMetrics = new ArrayList<>(); + List> processorMetrics = new ArrayList<>(); getProcessorMetrics(rootProcessor, processorMetrics); processorMetrics.forEach(t -> { Processor processor = t.v1(); - IngestMetric processorMetric = t.v2(); + OperationMetrics processorMetric = t.v2(); statsBuilder.addProcessorMetrics(id, getProcessorName(processor), processor.getType(), processorMetric); }); }); @@ -739,7 +740,7 @@ private void innerExecute( long startTimeInNanos = System.nanoTime(); // the pipeline specific stat holder may not exist and that is fine: // (e.g. the pipeline may have been removed while we're ingesting a document - totalMetrics.preIngest(); + totalMetrics.before(); String index = indexRequest.index(); String id = indexRequest.id(); String routing = indexRequest.routing(); @@ -749,9 +750,9 @@ private void innerExecute( IngestDocument ingestDocument = new IngestDocument(index, id, routing, version, versionType, sourceAsMap); ingestDocument.executePipeline(pipeline, (result, e) -> { long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeInNanos); - totalMetrics.postIngest(ingestTimeInMillis); + totalMetrics.after(ingestTimeInMillis); if (e != null) { - totalMetrics.ingestFailed(); + totalMetrics.failed(); handler.accept(e); } else if (result == null) { itemDroppedHandler.accept(slot); @@ -835,22 +836,22 @@ void innerUpdatePipelines(IngestMetadata newIngestMetadata) { } Pipeline oldPipeline = previous.pipeline; newPipeline.getMetrics().add(oldPipeline.getMetrics()); - List> oldPerProcessMetrics = new ArrayList<>(); - List> newPerProcessMetrics = new ArrayList<>(); + List> oldPerProcessMetrics = new ArrayList<>(); + List> newPerProcessMetrics = new ArrayList<>(); getProcessorMetrics(oldPipeline.getCompoundProcessor(), oldPerProcessMetrics); getProcessorMetrics(newPipeline.getCompoundProcessor(), newPerProcessMetrics); // Best attempt to populate new processor metrics using a parallel array of the old metrics. This is not ideal since // the per processor metrics may get reset when the arrays don't match. However, to get to an ideal model, unique and // consistent id's per processor and/or semantic equals for each processor will be needed. if (newPerProcessMetrics.size() == oldPerProcessMetrics.size()) { - Iterator> oldMetricsIterator = oldPerProcessMetrics.iterator(); - for (Tuple compositeMetric : newPerProcessMetrics) { + Iterator> oldMetricsIterator = oldPerProcessMetrics.iterator(); + for (Tuple compositeMetric : newPerProcessMetrics) { String type = compositeMetric.v1().getType(); - IngestMetric metric = compositeMetric.v2(); + OperationMetrics metric = compositeMetric.v2(); if (oldMetricsIterator.hasNext()) { - Tuple oldCompositeMetric = oldMetricsIterator.next(); + Tuple oldCompositeMetric = oldMetricsIterator.next(); String oldType = oldCompositeMetric.v1().getType(); - IngestMetric oldMetric = oldCompositeMetric.v2(); + OperationMetrics oldMetric = oldCompositeMetric.v2(); if (type.equals(oldType)) { metric.add(oldMetric); } diff --git a/server/src/main/java/org/opensearch/ingest/IngestStats.java b/server/src/main/java/org/opensearch/ingest/IngestStats.java index 0f6209d1c005e..ac06d779bdf2d 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestStats.java +++ b/server/src/main/java/org/opensearch/ingest/IngestStats.java @@ -35,7 +35,8 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.metrics.OperationMetrics; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -46,15 +47,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.TimeUnit; /** - * Stats for an ingest processor pipeline + * OperationStats for an ingest processor pipeline * * @opensearch.internal */ public class IngestStats implements Writeable, ToXContentFragment { - private final Stats totalStats; + private final OperationStats totalStats; private final List pipelineStats; private final Map> processorStats; @@ -64,7 +64,7 @@ public class IngestStats implements Writeable, ToXContentFragment { * @param pipelineStats - The stats for a given ingest pipeline. * @param processorStats - The per-processor stats for a given pipeline. A map keyed by the pipeline identifier. */ - public IngestStats(Stats totalStats, List pipelineStats, Map> processorStats) { + public IngestStats(OperationStats totalStats, List pipelineStats, Map> processorStats) { this.totalStats = totalStats; this.pipelineStats = pipelineStats; this.processorStats = processorStats; @@ -74,13 +74,13 @@ public IngestStats(Stats totalStats, List pipelineStats, Map(size); this.processorStats = new HashMap<>(size); for (int i = 0; i < size; i++) { String pipelineId = in.readString(); - Stats pipelineStat = new Stats(in); + OperationStats pipelineStat = new OperationStats(in); this.pipelineStats.add(new PipelineStat(pipelineId, pipelineStat)); int processorsSize = in.readVInt(); List processorStatsPerPipeline = new ArrayList<>(processorsSize); @@ -88,7 +88,7 @@ public IngestStats(StreamInput in) throws IOException { String processorName = in.readString(); String processorType = "_NOT_AVAILABLE"; processorType = in.readString(); - Stats processorStat = new Stats(in); + OperationStats processorStat = new OperationStats(in); processorStatsPerPipeline.add(new ProcessorStat(processorName, processorType, processorStat)); } this.processorStats.put(pipelineId, processorStatsPerPipeline); @@ -148,7 +148,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public Stats getTotalStats() { + public OperationStats getTotalStats() { return totalStats; } @@ -176,115 +176,24 @@ public int hashCode() { } /** - * The ingest statistics. - * - * @opensearch.internal - */ - public static class Stats implements Writeable, ToXContentFragment { - - private final long ingestCount; - private final long ingestTimeInMillis; - private final long ingestCurrent; - private final long ingestFailedCount; - - public Stats(long ingestCount, long ingestTimeInMillis, long ingestCurrent, long ingestFailedCount) { - this.ingestCount = ingestCount; - this.ingestTimeInMillis = ingestTimeInMillis; - this.ingestCurrent = ingestCurrent; - this.ingestFailedCount = ingestFailedCount; - } - - /** - * Read from a stream. - */ - public Stats(StreamInput in) throws IOException { - ingestCount = in.readVLong(); - ingestTimeInMillis = in.readVLong(); - ingestCurrent = in.readVLong(); - ingestFailedCount = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(ingestCount); - out.writeVLong(ingestTimeInMillis); - out.writeVLong(ingestCurrent); - out.writeVLong(ingestFailedCount); - } - - /** - * @return The total number of executed ingest preprocessing operations. - */ - public long getIngestCount() { - return ingestCount; - } - - /** - * @return The total time spent of ingest preprocessing in millis. - */ - public long getIngestTimeInMillis() { - return ingestTimeInMillis; - } - - /** - * @return The total number of ingest preprocessing operations currently executing. - */ - public long getIngestCurrent() { - return ingestCurrent; - } - - /** - * @return The total number of ingest preprocessing operations that have failed. - */ - public long getIngestFailedCount() { - return ingestFailedCount; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("count", ingestCount); - builder.humanReadableField("time_in_millis", "time", new TimeValue(ingestTimeInMillis, TimeUnit.MILLISECONDS)); - builder.field("current", ingestCurrent); - builder.field("failed", ingestFailedCount); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IngestStats.Stats that = (IngestStats.Stats) o; - return Objects.equals(ingestCount, that.ingestCount) - && Objects.equals(ingestTimeInMillis, that.ingestTimeInMillis) - && Objects.equals(ingestFailedCount, that.ingestFailedCount) - && Objects.equals(ingestCurrent, that.ingestCurrent); - } - - @Override - public int hashCode() { - return Objects.hash(ingestCount, ingestTimeInMillis, ingestFailedCount, ingestCurrent); - } - } - - /** - * Easy conversion from scoped {@link IngestMetric} objects to a serializable Stats objects + * Easy conversion from scoped {@link OperationMetrics} objects to a serializable OperationStats objects */ static class Builder { - private Stats totalStats; + private OperationStats totalStats; private List pipelineStats = new ArrayList<>(); private Map> processorStats = new HashMap<>(); - Builder addTotalMetrics(IngestMetric totalMetric) { + Builder addTotalMetrics(OperationMetrics totalMetric) { this.totalStats = totalMetric.createStats(); return this; } - Builder addPipelineMetrics(String pipelineId, IngestMetric pipelineMetric) { + Builder addPipelineMetrics(String pipelineId, OperationMetrics pipelineMetric) { this.pipelineStats.add(new PipelineStat(pipelineId, pipelineMetric.createStats())); return this; } - Builder addProcessorMetrics(String pipelineId, String processorName, String processorType, IngestMetric metric) { + Builder addProcessorMetrics(String pipelineId, String processorName, String processorType, OperationMetrics metric) { this.processorStats.computeIfAbsent(pipelineId, k -> new ArrayList<>()) .add(new ProcessorStat(processorName, processorType, metric.createStats())); return this; @@ -300,9 +209,9 @@ IngestStats build() { */ public static class PipelineStat { private final String pipelineId; - private final Stats stats; + private final OperationStats stats; - public PipelineStat(String pipelineId, Stats stats) { + public PipelineStat(String pipelineId, OperationStats stats) { this.pipelineId = pipelineId; this.stats = stats; } @@ -311,7 +220,7 @@ public String getPipelineId() { return pipelineId; } - public Stats getStats() { + public OperationStats getStats() { return stats; } @@ -335,9 +244,9 @@ public int hashCode() { public static class ProcessorStat { private final String name; private final String type; - private final Stats stats; + private final OperationStats stats; - public ProcessorStat(String name, String type, Stats stats) { + public ProcessorStat(String name, String type, OperationStats stats) { this.name = name; this.type = type; this.stats = stats; @@ -351,7 +260,7 @@ public String getType() { return type; } - public Stats getStats() { + public OperationStats getStats() { return stats; } diff --git a/server/src/main/java/org/opensearch/ingest/Pipeline.java b/server/src/main/java/org/opensearch/ingest/Pipeline.java index 9b3725fd65d9d..766fb9cd66777 100644 --- a/server/src/main/java/org/opensearch/ingest/Pipeline.java +++ b/server/src/main/java/org/opensearch/ingest/Pipeline.java @@ -43,6 +43,7 @@ import java.util.function.BiConsumer; import java.util.function.LongSupplier; +import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.script.ScriptService; /** @@ -63,7 +64,7 @@ public final class Pipeline { @Nullable private final Integer version; private final CompoundProcessor compoundProcessor; - private final IngestMetric metrics; + private final OperationMetrics metrics; private final LongSupplier relativeTimeProvider; public Pipeline(String id, @Nullable String description, @Nullable Integer version, CompoundProcessor compoundProcessor) { @@ -82,7 +83,7 @@ public Pipeline(String id, @Nullable String description, @Nullable Integer versi this.description = description; this.compoundProcessor = compoundProcessor; this.version = version; - this.metrics = new IngestMetric(); + this.metrics = new OperationMetrics(); this.relativeTimeProvider = relativeTimeProvider; } @@ -129,12 +130,12 @@ public static Pipeline create( */ public void execute(IngestDocument ingestDocument, BiConsumer handler) { final long startTimeInNanos = relativeTimeProvider.getAsLong(); - metrics.preIngest(); + metrics.before(); compoundProcessor.execute(ingestDocument, (result, e) -> { long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - startTimeInNanos); - metrics.postIngest(ingestTimeInMillis); + metrics.after(ingestTimeInMillis); if (e != null) { - metrics.ingestFailed(); + metrics.failed(); } handler.accept(result, e); }); @@ -198,7 +199,7 @@ public List flattenAllProcessors() { /** * The metrics associated with this pipeline. */ - public IngestMetric getMetrics() { + public OperationMetrics getMetrics() { return metrics; } } diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index 9382746081c18..6f4fe1e083ad7 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -216,7 +216,8 @@ public NodeStats stats( boolean clusterManagerThrottling, boolean weightedRoutingStats, boolean fileCacheStats, - boolean taskCancellation + boolean taskCancellation, + boolean searchPipelineStats ) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) @@ -243,7 +244,8 @@ public NodeStats stats( clusterManagerThrottling ? this.clusterService.getClusterManagerService().getThrottlingStats() : null, weightedRoutingStats ? WeightedRoutingStats.getInstance() : null, fileCacheStats && fileCache != null ? fileCache.fileCacheStats() : null, - taskCancellation ? this.taskCancellationMonitoringService.stats() : null + taskCancellation ? this.taskCancellationMonitoringService.stats() : null, + searchPipelineStats ? this.searchPipelineService.stats() : null ); } diff --git a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java index c9a5f865d507e..6f44daf48ed21 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java +++ b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java @@ -8,7 +8,6 @@ package org.opensearch.search.pipeline; -import org.opensearch.OpenSearchParseException; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Nullable; @@ -16,17 +15,11 @@ import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.ingest.ConfigurationUtils; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; - -import static org.opensearch.ingest.ConfigurationUtils.TAG_KEY; -import static org.opensearch.ingest.Pipeline.DESCRIPTION_KEY; -import static org.opensearch.ingest.Pipeline.VERSION_KEY; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; /** * Concrete representation of a search pipeline, holding multiple processors. @@ -45,73 +38,24 @@ class Pipeline { private final List searchResponseProcessors; private final NamedWriteableRegistry namedWriteableRegistry; + private final LongSupplier relativeTimeSupplier; - private Pipeline( + Pipeline( String id, @Nullable String description, @Nullable Integer version, List requestProcessors, List responseProcessors, - NamedWriteableRegistry namedWriteableRegistry + NamedWriteableRegistry namedWriteableRegistry, + LongSupplier relativeTimeSupplier ) { this.id = id; this.description = description; this.version = version; - this.searchRequestProcessors = requestProcessors; - this.searchResponseProcessors = responseProcessors; + this.searchRequestProcessors = Collections.unmodifiableList(requestProcessors); + this.searchResponseProcessors = Collections.unmodifiableList(responseProcessors); this.namedWriteableRegistry = namedWriteableRegistry; - } - - static Pipeline create( - String id, - Map config, - Map> requestProcessorFactories, - Map> responseProcessorFactories, - NamedWriteableRegistry namedWriteableRegistry - ) throws Exception { - String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY); - Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null); - List> requestProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, REQUEST_PROCESSORS_KEY); - List requestProcessors = readProcessors(requestProcessorFactories, requestProcessorConfigs); - List> responseProcessorConfigs = ConfigurationUtils.readOptionalList( - null, - null, - config, - RESPONSE_PROCESSORS_KEY - ); - List responseProcessors = readProcessors(responseProcessorFactories, responseProcessorConfigs); - if (config.isEmpty() == false) { - throw new OpenSearchParseException( - "pipeline [" - + id - + "] doesn't support one or more provided configuration parameters " - + Arrays.toString(config.keySet().toArray()) - ); - } - return new Pipeline(id, description, version, requestProcessors, responseProcessors, namedWriteableRegistry); - } - - private static List readProcessors( - Map> processorFactories, - List> requestProcessorConfigs - ) throws Exception { - List processors = new ArrayList<>(); - if (requestProcessorConfigs == null) { - return processors; - } - for (Map processorConfigWithKey : requestProcessorConfigs) { - for (Map.Entry entry : processorConfigWithKey.entrySet()) { - String type = entry.getKey(); - if (!processorFactories.containsKey(type)) { - throw new IllegalArgumentException("Invalid processor type " + type); - } - Map config = (Map) entry.getValue(); - String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); - String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); - processors.add(processorFactories.get(type).create(processorFactories, tag, description, config)); - } - } - return Collections.unmodifiableList(processors); + this.relativeTimeSupplier = relativeTimeSupplier; } String getId() { @@ -134,32 +78,94 @@ List getSearchResponseProcessors() { return searchResponseProcessors; } - SearchRequest transformRequest(SearchRequest request) throws Exception { + protected void beforeTransformRequest() {} + + protected void afterTransformRequest(long timeInNanos) {} + + protected void onTransformRequestFailure() {} + + protected void beforeRequestProcessor(Processor processor) {} + + protected void afterRequestProcessor(Processor processor, long timeInNanos) {} + + protected void onRequestProcessorFailed(Processor processor) {} + + protected void beforeTransformResponse() {} + + protected void afterTransformResponse(long timeInNanos) {} + + protected void onTransformResponseFailure() {} + + protected void beforeResponseProcessor(Processor processor) {} + + protected void afterResponseProcessor(Processor processor, long timeInNanos) {} + + protected void onResponseProcessorFailed(Processor processor) {} + + SearchRequest transformRequest(SearchRequest request) throws SearchPipelineProcessingException { if (searchRequestProcessors.isEmpty() == false) { - try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { - request.writeTo(bytesStreamOutput); - try (StreamInput in = bytesStreamOutput.bytes().streamInput()) { - try (StreamInput input = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry)) { - request = new SearchRequest(input); + long pipelineStart = relativeTimeSupplier.getAsLong(); + beforeTransformRequest(); + try { + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + request.writeTo(bytesStreamOutput); + try (StreamInput in = bytesStreamOutput.bytes().streamInput()) { + try (StreamInput input = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry)) { + request = new SearchRequest(input); + } } } - } - for (SearchRequestProcessor searchRequestProcessor : searchRequestProcessors) { - request = searchRequestProcessor.processRequest(request); + for (SearchRequestProcessor processor : searchRequestProcessors) { + beforeRequestProcessor(processor); + long start = relativeTimeSupplier.getAsLong(); + try { + request = processor.processRequest(request); + } catch (Exception e) { + onRequestProcessorFailed(processor); + throw e; + } finally { + long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start); + afterRequestProcessor(processor, took); + } + } + } catch (Exception e) { + onTransformRequestFailure(); + throw new SearchPipelineProcessingException(e); + } finally { + long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart); + afterTransformRequest(took); } } return request; } SearchResponse transformResponse(SearchRequest request, SearchResponse response) throws SearchPipelineProcessingException { - try { - for (SearchResponseProcessor responseProcessor : searchResponseProcessors) { - response = responseProcessor.processResponse(request, response); + if (searchResponseProcessors.isEmpty() == false) { + long pipelineStart = relativeTimeSupplier.getAsLong(); + beforeTransformResponse(); + try { + for (SearchResponseProcessor processor : searchResponseProcessors) { + beforeResponseProcessor(processor); + long start = relativeTimeSupplier.getAsLong(); + try { + response = processor.processResponse(request, response); + } catch (Exception e) { + onResponseProcessorFailed(processor); + throw e; + } finally { + long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start); + afterResponseProcessor(processor, took); + } + } + } catch (Exception e) { + onTransformResponseFailure(); + throw new SearchPipelineProcessingException(e); + } finally { + long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart); + afterTransformResponse(took); } - return response; - } catch (Exception e) { - throw new SearchPipelineProcessingException(e); } + return response; } static final Pipeline NO_OP_PIPELINE = new Pipeline( @@ -168,6 +174,8 @@ SearchResponse transformResponse(SearchRequest request, SearchResponse response) 0, Collections.emptyList(), Collections.emptyList(), - null + null, + () -> 0L ); + } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java new file mode 100644 index 0000000000000..662473f190006 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java @@ -0,0 +1,227 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.common.metrics.OperationMetrics; +import org.opensearch.ingest.ConfigurationUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.LongSupplier; + +import static org.opensearch.ingest.ConfigurationUtils.TAG_KEY; +import static org.opensearch.ingest.Pipeline.DESCRIPTION_KEY; +import static org.opensearch.ingest.Pipeline.VERSION_KEY; + +/** + * Specialization of {@link Pipeline} that adds metrics to track executions of the pipeline and individual processors. + */ +class PipelineWithMetrics extends Pipeline { + + private final OperationMetrics totalRequestMetrics; + private final OperationMetrics totalResponseMetrics; + private final OperationMetrics pipelineRequestMetrics = new OperationMetrics(); + private final OperationMetrics pipelineResponseMetrics = new OperationMetrics(); + private final Map requestProcessorMetrics = new HashMap<>(); + private final Map responseProcessorMetrics = new HashMap<>(); + + PipelineWithMetrics( + String id, + String description, + Integer version, + List requestProcessors, + List responseProcessors, + NamedWriteableRegistry namedWriteableRegistry, + OperationMetrics totalRequestMetrics, + OperationMetrics totalResponseMetrics, + LongSupplier relativeTimeSupplier + ) { + super(id, description, version, requestProcessors, responseProcessors, namedWriteableRegistry, relativeTimeSupplier); + this.totalRequestMetrics = totalRequestMetrics; + this.totalResponseMetrics = totalResponseMetrics; + for (Processor requestProcessor : getSearchRequestProcessors()) { + requestProcessorMetrics.putIfAbsent(getProcessorKey(requestProcessor), new OperationMetrics()); + } + for (Processor responseProcessor : getSearchResponseProcessors()) { + responseProcessorMetrics.putIfAbsent(getProcessorKey(responseProcessor), new OperationMetrics()); + } + } + + static PipelineWithMetrics create( + String id, + Map config, + Map> requestProcessorFactories, + Map> responseProcessorFactories, + NamedWriteableRegistry namedWriteableRegistry, + OperationMetrics totalRequestProcessingMetrics, + OperationMetrics totalResponseProcessingMetrics + ) throws Exception { + String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY); + Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null); + List> requestProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, REQUEST_PROCESSORS_KEY); + List requestProcessors = readProcessors(requestProcessorFactories, requestProcessorConfigs); + List> responseProcessorConfigs = ConfigurationUtils.readOptionalList( + null, + null, + config, + RESPONSE_PROCESSORS_KEY + ); + List responseProcessors = readProcessors(responseProcessorFactories, responseProcessorConfigs); + if (config.isEmpty() == false) { + throw new OpenSearchParseException( + "pipeline [" + + id + + "] doesn't support one or more provided configuration parameters " + + Arrays.toString(config.keySet().toArray()) + ); + } + return new PipelineWithMetrics( + id, + description, + version, + requestProcessors, + responseProcessors, + namedWriteableRegistry, + totalRequestProcessingMetrics, + totalResponseProcessingMetrics, + System::nanoTime + ); + + } + + private static List readProcessors( + Map> processorFactories, + List> requestProcessorConfigs + ) throws Exception { + List processors = new ArrayList<>(); + if (requestProcessorConfigs == null) { + return processors; + } + for (Map processorConfigWithKey : requestProcessorConfigs) { + for (Map.Entry entry : processorConfigWithKey.entrySet()) { + String type = entry.getKey(); + if (!processorFactories.containsKey(type)) { + throw new IllegalArgumentException("Invalid processor type " + type); + } + Map config = (Map) entry.getValue(); + String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); + String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); + processors.add(processorFactories.get(type).create(processorFactories, tag, description, config)); + } + } + return Collections.unmodifiableList(processors); + } + + @Override + protected void beforeTransformRequest() { + super.beforeTransformRequest(); + totalRequestMetrics.before(); + pipelineRequestMetrics.before(); + } + + @Override + protected void afterTransformRequest(long timeInNanos) { + super.afterTransformRequest(timeInNanos); + totalRequestMetrics.after(timeInNanos); + pipelineRequestMetrics.after(timeInNanos); + } + + @Override + protected void onTransformRequestFailure() { + super.onTransformRequestFailure(); + totalRequestMetrics.failed(); + pipelineRequestMetrics.failed(); + } + + protected void beforeRequestProcessor(Processor processor) { + requestProcessorMetrics.get(getProcessorKey(processor)).before(); + } + + protected void afterRequestProcessor(Processor processor, long timeInNanos) { + requestProcessorMetrics.get(getProcessorKey(processor)).after(timeInNanos); + } + + protected void onRequestProcessorFailed(Processor processor) { + requestProcessorMetrics.get(getProcessorKey(processor)).failed(); + } + + protected void beforeTransformResponse() { + super.beforeTransformRequest(); + totalResponseMetrics.before(); + pipelineResponseMetrics.before(); + } + + protected void afterTransformResponse(long timeInNanos) { + super.afterTransformResponse(timeInNanos); + totalResponseMetrics.after(timeInNanos); + pipelineResponseMetrics.after(timeInNanos); + } + + protected void onTransformResponseFailure() { + super.onTransformResponseFailure(); + totalResponseMetrics.failed(); + pipelineResponseMetrics.failed(); + } + + protected void beforeResponseProcessor(Processor processor) { + responseProcessorMetrics.get(getProcessorKey(processor)).before(); + } + + protected void afterResponseProcessor(Processor processor, long timeInNanos) { + responseProcessorMetrics.get(getProcessorKey(processor)).after(timeInNanos); + } + + protected void onResponseProcessorFailed(Processor processor) { + responseProcessorMetrics.get(getProcessorKey(processor)).failed(); + } + + void copyMetrics(PipelineWithMetrics oldPipeline) { + pipelineRequestMetrics.add(oldPipeline.pipelineRequestMetrics); + pipelineResponseMetrics.add(oldPipeline.pipelineResponseMetrics); + copyProcessorMetrics(requestProcessorMetrics, oldPipeline.requestProcessorMetrics); + copyProcessorMetrics(responseProcessorMetrics, oldPipeline.responseProcessorMetrics); + } + + private static void copyProcessorMetrics( + Map newProcessorMetrics, + Map oldProcessorMetrics + ) { + for (Map.Entry oldProcessorMetric : oldProcessorMetrics.entrySet()) { + if (newProcessorMetrics.containsKey(oldProcessorMetric.getKey())) { + newProcessorMetrics.get(oldProcessorMetric.getKey()).add(oldProcessorMetric.getValue()); + } + } + } + + private static String getProcessorKey(Processor processor) { + String key = processor.getType(); + if (processor.getTag() != null) { + return key + ":" + processor.getTag(); + } + return key; + } + + void populateStats(SearchPipelineStats.Builder statsBuilder) { + statsBuilder.addPipelineStats(getId(), pipelineRequestMetrics, pipelineResponseMetrics); + for (Processor processor : getSearchRequestProcessors()) { + String key = getProcessorKey(processor); + statsBuilder.addRequestProcessorStats(getId(), key, processor.getType(), requestProcessorMetrics.get(key)); + } + for (Processor processor : getSearchResponseProcessors()) { + String key = getProcessorKey(processor); + statsBuilder.addResponseProcessorStats(getId(), key, processor.getType(), responseProcessorMetrics.get(key)); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 87c09bd971284..434c8fbfacc74 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -30,6 +30,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -80,6 +81,9 @@ public class SearchPipelineService implements ClusterStateApplier, ReportingServ private final NamedWriteableRegistry namedWriteableRegistry; private volatile ClusterState state; + private final OperationMetrics totalRequestProcessingMetrics = new OperationMetrics(); + private final OperationMetrics totalResponseProcessingMetrics = new OperationMetrics(); + private final boolean isEnabled; public SearchPipelineService( @@ -172,26 +176,26 @@ void innerUpdatePipelines(SearchPipelineMetadata newSearchPipelineMetadata) { newPipelines = new HashMap<>(existingPipelines); } try { - Pipeline newPipeline = Pipeline.create( + PipelineWithMetrics newPipeline = PipelineWithMetrics.create( newConfiguration.getId(), newConfiguration.getConfigAsMap(), requestProcessorFactories, responseProcessorFactories, - namedWriteableRegistry + namedWriteableRegistry, + totalRequestProcessingMetrics, + totalResponseProcessingMetrics ); newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, newPipeline)); - if (previous == null) { - continue; + if (previous != null) { + newPipeline.copyMetrics(previous.pipeline); } - // TODO -- once we add in pipeline metrics (like in ingest pipelines), we will need to deep-copy - // the old pipeline's metrics into the new pipeline. } catch (Exception e) { OpenSearchParseException parseException = new OpenSearchParseException( "Error updating pipeline with id [" + newConfiguration.getId() + "]", e ); - // TODO -- replace pipeline with one that throws an exception when we try to use it + // TODO -- replace pipeline with one that throws this exception when we try to use it if (exceptions == null) { exceptions = new ArrayList<>(); } @@ -271,12 +275,14 @@ void validatePipeline(Map searchPipelineInfos throw new IllegalStateException("Search pipeline info is empty"); } Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); - Pipeline pipeline = Pipeline.create( + Pipeline pipeline = PipelineWithMetrics.create( request.getId(), pipelineConfig, requestProcessorFactories, responseProcessorFactories, - namedWriteableRegistry + namedWriteableRegistry, + new OperationMetrics(), // Use ephemeral metrics for validation + new OperationMetrics() ); List exceptions = new ArrayList<>(); for (SearchRequestProcessor processor : pipeline.getSearchRequestProcessors()) { @@ -367,12 +373,14 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) throws Exce ); } try { - pipeline = Pipeline.create( + pipeline = PipelineWithMetrics.create( AD_HOC_PIPELINE_ID, searchRequest.source().searchPipelineSource(), requestProcessorFactories, responseProcessorFactories, - namedWriteableRegistry + namedWriteableRegistry, + totalRequestProcessingMetrics, + totalResponseProcessingMetrics ); } catch (Exception e) { throw new SearchPipelineProcessingException(e); @@ -400,12 +408,8 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) throws Exce pipeline = pipelineHolder.pipeline; } } - try { - SearchRequest transformedRequest = pipeline.transformRequest(searchRequest); - return new PipelinedRequest(pipeline, transformedRequest); - } catch (Exception e) { - throw new SearchPipelineProcessingException(e); - } + SearchRequest transformedRequest = pipeline.transformRequest(searchRequest); + return new PipelinedRequest(pipeline, transformedRequest); } Map> getRequestProcessorFactories() { @@ -431,6 +435,16 @@ public SearchPipelineInfo info() { ); } + public SearchPipelineStats stats() { + SearchPipelineStats.Builder builder = new SearchPipelineStats.Builder(); + builder.withTotalStats(totalRequestProcessingMetrics, totalResponseProcessingMetrics); + for (PipelineHolder pipelineHolder : pipelines.values()) { + PipelineWithMetrics pipeline = pipelineHolder.pipeline; + pipeline.populateStats(builder); + } + return builder.build(); + } + public static List getPipelines(ClusterState clusterState, String... ids) { SearchPipelineMetadata metadata = clusterState.getMetadata().custom(SearchPipelineMetadata.TYPE); return innerGetPipelines(metadata, ids); @@ -474,9 +488,9 @@ Map getPipelines() { static class PipelineHolder { final PipelineConfiguration configuration; - final Pipeline pipeline; + final PipelineWithMetrics pipeline; - PipelineHolder(PipelineConfiguration configuration, Pipeline pipeline) { + PipelineHolder(PipelineConfiguration configuration, PipelineWithMetrics pipeline) { this.configuration = Objects.requireNonNull(configuration); this.pipeline = Objects.requireNonNull(pipeline); } diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java new file mode 100644 index 0000000000000..4261bfe99160a --- /dev/null +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java @@ -0,0 +1,367 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline; + +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.metrics.OperationMetrics; +import org.opensearch.common.metrics.OperationStats; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; + +/** + * Serializable, immutable search pipeline statistics to be returned via stats APIs. + * + * @opensearch.internal + */ +public class SearchPipelineStats implements Writeable, ToXContentFragment { + + private final OperationStats totalRequestStats; + private final OperationStats totalResponseStats; + private final List perPipelineStats; + private final Map perPipelineProcessorStats; + + public SearchPipelineStats( + OperationStats totalRequestStats, + OperationStats totalResponseStats, + List perPipelineStats, + Map perPipelineProcessorStats + ) { + this.totalRequestStats = totalRequestStats; + this.totalResponseStats = totalResponseStats; + this.perPipelineStats = perPipelineStats; + this.perPipelineProcessorStats = perPipelineProcessorStats; + } + + public SearchPipelineStats(StreamInput in) throws IOException { + this.totalRequestStats = new OperationStats(in); + this.totalResponseStats = new OperationStats(in); + int size = in.readVInt(); + List perPipelineStats = new ArrayList<>(size); + Map pipelineDetailStatsMap = new TreeMap<>(); + for (int i = 0; i < size; i++) { + String pipelineId = in.readString(); + OperationStats pipelineRequestStats = new OperationStats(in); + OperationStats pipelineResponseStats = new OperationStats(in); + perPipelineStats.add(new PerPipelineStats(pipelineId, pipelineRequestStats, pipelineResponseStats)); + int numRequestProcessors = in.readVInt(); + List requestProcessorStats = new ArrayList<>(numRequestProcessors); + for (int j = 0; j < numRequestProcessors; j++) { + String processorName = in.readString(); + String processorType = in.readString(); + OperationStats processorStats = new OperationStats(in); + requestProcessorStats.add(new ProcessorStats(processorName, processorType, processorStats)); + } + int numResponseProcessors = in.readVInt(); + List responseProcessorStats = new ArrayList<>(numResponseProcessors); + for (int j = 0; j < numResponseProcessors; j++) { + String processorName = in.readString(); + String processorType = in.readString(); + OperationStats processorStats = new OperationStats(in); + responseProcessorStats.add(new ProcessorStats(processorName, processorType, processorStats)); + } + pipelineDetailStatsMap.put( + pipelineId, + new PipelineDetailStats(unmodifiableList(requestProcessorStats), unmodifiableList(responseProcessorStats)) + ); + } + this.perPipelineStats = unmodifiableList(perPipelineStats); + this.perPipelineProcessorStats = unmodifiableMap(pipelineDetailStatsMap); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("search_pipeline"); + builder.startObject("total_request"); + totalRequestStats.toXContent(builder, params); + builder.endObject(); + builder.startObject("total_response"); + totalResponseStats.toXContent(builder, params); + builder.endObject(); + builder.startObject("pipelines"); + for (PerPipelineStats pipelineStat : perPipelineStats) { + builder.startObject(pipelineStat.pipelineId); + builder.startObject("request"); + pipelineStat.requestStats.toXContent(builder, params); + builder.endObject(); + builder.startObject("response"); + pipelineStat.responseStats.toXContent(builder, params); + builder.endObject(); + + PipelineDetailStats pipelineDetailStats = perPipelineProcessorStats.get(pipelineStat.pipelineId); + builder.startArray("request_processors"); + for (ProcessorStats processorStats : pipelineDetailStats.requestProcessorStats) { + builder.startObject(); + processorStats.toXContent(builder, params); + builder.endObject(); + } + builder.endArray(); + builder.startArray("response_processors"); + for (ProcessorStats processorStats : pipelineDetailStats.responseProcessorStats) { + builder.startObject(); + processorStats.toXContent(builder, params); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + } + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + totalRequestStats.writeTo(out); + totalResponseStats.writeTo(out); + out.writeVInt(perPipelineStats.size()); + for (PerPipelineStats pipelineStat : perPipelineStats) { + out.writeString(pipelineStat.pipelineId); + pipelineStat.requestStats.writeTo(out); + pipelineStat.responseStats.writeTo(out); + PipelineDetailStats pipelineDetailStats = perPipelineProcessorStats.get(pipelineStat.pipelineId); + out.writeVInt(pipelineDetailStats.requestProcessorStats.size()); + for (ProcessorStats processorStats : pipelineDetailStats.requestProcessorStats) { + out.writeString(processorStats.processorName); + out.writeString(processorStats.processorType); + processorStats.stats.writeTo(out); + } + out.writeVInt(pipelineDetailStats.responseProcessorStats.size()); + for (ProcessorStats processorStats : pipelineDetailStats.responseProcessorStats) { + out.writeString(processorStats.processorName); + out.writeString(processorStats.processorType); + processorStats.stats.writeTo(out); + } + } + } + + static class Builder { + private OperationStats totalRequestStats; + private OperationStats totalResponseStats; + private final List perPipelineStats = new ArrayList<>(); + private final Map> requestProcessorStatsPerPipeline = new HashMap<>(); + private final Map> responseProcessorStatsPerPipeline = new HashMap<>(); + + Builder withTotalStats(OperationMetrics totalRequestMetrics, OperationMetrics totalResponseMetrics) { + this.totalRequestStats = totalRequestMetrics.createStats(); + this.totalResponseStats = totalResponseMetrics.createStats(); + return this; + } + + Builder addPipelineStats(String pipelineId, OperationMetrics pipelineRequestMetrics, OperationMetrics pipelineResponseMetrics) { + this.perPipelineStats.add( + new PerPipelineStats(pipelineId, pipelineRequestMetrics.createStats(), pipelineResponseMetrics.createStats()) + ); + return this; + } + + Builder addRequestProcessorStats(String pipelineId, String processorName, String processorType, OperationMetrics processorMetrics) { + this.requestProcessorStatsPerPipeline.computeIfAbsent(pipelineId, k -> new ArrayList<>()) + .add(new ProcessorStats(processorName, processorType, processorMetrics.createStats())); + return this; + } + + Builder addResponseProcessorStats( + String pipelineId, + String processorName, + String processorType, + OperationMetrics processorMetrics + ) { + this.responseProcessorStatsPerPipeline.computeIfAbsent(pipelineId, k -> new ArrayList<>()) + .add(new ProcessorStats(processorName, processorType, processorMetrics.createStats())); + return this; + } + + SearchPipelineStats build() { + Map pipelineDetailStatsMap = new TreeMap<>(); + for (PerPipelineStats pipelineStat : perPipelineStats) { + List requestProcessorStats = requestProcessorStatsPerPipeline.getOrDefault( + pipelineStat.pipelineId, + emptyList() + ); + List responseProcessorStats = responseProcessorStatsPerPipeline.getOrDefault( + pipelineStat.pipelineId, + emptyList() + ); + PipelineDetailStats pipelineDetailStats = new PipelineDetailStats( + unmodifiableList(requestProcessorStats), + unmodifiableList(responseProcessorStats) + ); + pipelineDetailStatsMap.put(pipelineStat.pipelineId, pipelineDetailStats); + } + return new SearchPipelineStats( + totalRequestStats, + totalResponseStats, + unmodifiableList(perPipelineStats), + unmodifiableMap(pipelineDetailStatsMap) + ); + } + } + + static class PerPipelineStats { + private final String pipelineId; + private final OperationStats requestStats; + private final OperationStats responseStats; + + public PerPipelineStats(String pipelineId, OperationStats requestStats, OperationStats responseStats) { + this.pipelineId = pipelineId; + this.requestStats = requestStats; + this.responseStats = responseStats; + } + + public String getPipelineId() { + return pipelineId; + } + + public OperationStats getRequestStats() { + return requestStats; + } + + public OperationStats getResponseStats() { + return responseStats; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PerPipelineStats that = (PerPipelineStats) o; + return pipelineId.equals(that.pipelineId) && requestStats.equals(that.requestStats) && responseStats.equals(that.responseStats); + } + + @Override + public int hashCode() { + return Objects.hash(pipelineId, requestStats, responseStats); + } + } + + static class PipelineDetailStats { + private final List requestProcessorStats; + private final List responseProcessorStats; + + public PipelineDetailStats(List requestProcessorStats, List responseProcessorStats) { + this.requestProcessorStats = requestProcessorStats; + this.responseProcessorStats = responseProcessorStats; + } + + public List requestProcessorStats() { + return requestProcessorStats; + } + + public List responseProcessorStats() { + return responseProcessorStats; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PipelineDetailStats that = (PipelineDetailStats) o; + return requestProcessorStats.equals(that.requestProcessorStats) && responseProcessorStats.equals(that.responseProcessorStats); + } + + @Override + public int hashCode() { + return Objects.hash(requestProcessorStats, responseProcessorStats); + } + } + + static class ProcessorStats implements ToXContentFragment { + private final String processorName; // type:tag + private final String processorType; + private final OperationStats stats; + + public ProcessorStats(String processorName, String processorType, OperationStats stats) { + this.processorName = processorName; + this.processorType = processorType; + this.stats = stats; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ProcessorStats that = (ProcessorStats) o; + return processorName.equals(that.processorName) && processorType.equals(that.processorType) && stats.equals(that.stats); + } + + @Override + public int hashCode() { + return Objects.hash(processorName, processorType, stats); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(processorName); + builder.field("type", processorType); + builder.startObject("stats"); + stats.toXContent(builder, params); + builder.endObject(); + builder.endObject(); + return builder; + } + + String getProcessorName() { + return processorName; + } + + String getProcessorType() { + return processorType; + } + + OperationStats getStats() { + return stats; + } + } + + OperationStats getTotalRequestStats() { + return totalRequestStats; + } + + OperationStats getTotalResponseStats() { + return totalResponseStats; + } + + List getPipelineStats() { + return perPipelineStats; + } + + Map getPerPipelineProcessorStats() { + return perPipelineProcessorStats; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SearchPipelineStats stats = (SearchPipelineStats) o; + return totalRequestStats.equals(stats.totalRequestStats) + && totalResponseStats.equals(stats.totalResponseStats) + && perPipelineStats.equals(stats.perPipelineStats) + && perPipelineProcessorStats.equals(stats.perPipelineProcessorStats); + } + + @Override + public int hashCode() { + return Objects.hash(totalRequestStats, totalResponseStats, perPipelineStats, perPipelineProcessorStats); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index d99b93b780140..1e2e085333e50 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -37,6 +37,7 @@ import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.discovery.DiscoveryStats; import org.opensearch.cluster.coordination.PendingClusterStateStats; import org.opensearch.cluster.coordination.PublishClusterStateStats; @@ -338,40 +339,31 @@ public void testSerialization() throws IOException { if (ingestStats == null) { assertNull(deserializedIngestStats); } else { - IngestStats.Stats totalStats = ingestStats.getTotalStats(); - assertEquals(totalStats.getIngestCount(), deserializedIngestStats.getTotalStats().getIngestCount()); - assertEquals(totalStats.getIngestCurrent(), deserializedIngestStats.getTotalStats().getIngestCurrent()); - assertEquals(totalStats.getIngestFailedCount(), deserializedIngestStats.getTotalStats().getIngestFailedCount()); - assertEquals(totalStats.getIngestTimeInMillis(), deserializedIngestStats.getTotalStats().getIngestTimeInMillis()); + OperationStats totalStats = ingestStats.getTotalStats(); + assertEquals(totalStats.getCount(), deserializedIngestStats.getTotalStats().getCount()); + assertEquals(totalStats.getCurrent(), deserializedIngestStats.getTotalStats().getCurrent()); + assertEquals(totalStats.getFailedCount(), deserializedIngestStats.getTotalStats().getFailedCount()); + assertEquals(totalStats.getTotalTimeInMillis(), deserializedIngestStats.getTotalStats().getTotalTimeInMillis()); assertEquals(ingestStats.getPipelineStats().size(), deserializedIngestStats.getPipelineStats().size()); for (IngestStats.PipelineStat pipelineStat : ingestStats.getPipelineStats()) { String pipelineId = pipelineStat.getPipelineId(); - IngestStats.Stats deserializedPipelineStats = getPipelineStats( - deserializedIngestStats.getPipelineStats(), - pipelineId - ); - assertEquals(pipelineStat.getStats().getIngestFailedCount(), deserializedPipelineStats.getIngestFailedCount()); - assertEquals(pipelineStat.getStats().getIngestTimeInMillis(), deserializedPipelineStats.getIngestTimeInMillis()); - assertEquals(pipelineStat.getStats().getIngestCurrent(), deserializedPipelineStats.getIngestCurrent()); - assertEquals(pipelineStat.getStats().getIngestCount(), deserializedPipelineStats.getIngestCount()); + OperationStats deserializedPipelineStats = getPipelineStats(deserializedIngestStats.getPipelineStats(), pipelineId); + assertEquals(pipelineStat.getStats().getFailedCount(), deserializedPipelineStats.getFailedCount()); + assertEquals(pipelineStat.getStats().getTotalTimeInMillis(), deserializedPipelineStats.getTotalTimeInMillis()); + assertEquals(pipelineStat.getStats().getCurrent(), deserializedPipelineStats.getCurrent()); + assertEquals(pipelineStat.getStats().getCount(), deserializedPipelineStats.getCount()); List processorStats = ingestStats.getProcessorStats().get(pipelineId); // intentionally validating identical order Iterator it = deserializedIngestStats.getProcessorStats().get(pipelineId).iterator(); for (IngestStats.ProcessorStat processorStat : processorStats) { IngestStats.ProcessorStat deserializedProcessorStat = it.next(); + assertEquals(processorStat.getStats().getFailedCount(), deserializedProcessorStat.getStats().getFailedCount()); assertEquals( - processorStat.getStats().getIngestFailedCount(), - deserializedProcessorStat.getStats().getIngestFailedCount() - ); - assertEquals( - processorStat.getStats().getIngestTimeInMillis(), - deserializedProcessorStat.getStats().getIngestTimeInMillis() + processorStat.getStats().getTotalTimeInMillis(), + deserializedProcessorStat.getStats().getTotalTimeInMillis() ); - assertEquals( - processorStat.getStats().getIngestCurrent(), - deserializedProcessorStat.getStats().getIngestCurrent() - ); - assertEquals(processorStat.getStats().getIngestCount(), deserializedProcessorStat.getStats().getIngestCount()); + assertEquals(processorStat.getStats().getCurrent(), deserializedProcessorStat.getStats().getCurrent()); + assertEquals(processorStat.getStats().getCount(), deserializedProcessorStat.getStats().getCount()); } assertFalse(it.hasNext()); } @@ -650,7 +642,7 @@ public static NodeStats createNodeStats() { : null; IngestStats ingestStats = null; if (frequently()) { - IngestStats.Stats totalStats = new IngestStats.Stats( + OperationStats totalStats = new OperationStats( randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), @@ -665,7 +657,7 @@ public static NodeStats createNodeStats() { ingestPipelineStats.add( new IngestStats.PipelineStat( pipelineId, - new IngestStats.Stats( + new OperationStats( randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), @@ -676,7 +668,7 @@ public static NodeStats createNodeStats() { List processorPerPipeline = new ArrayList<>(numProcessors); for (int j = 0; j < numProcessors; j++) { - IngestStats.Stats processorStats = new IngestStats.Stats( + OperationStats processorStats = new OperationStats( randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), @@ -750,11 +742,12 @@ public static NodeStats createNodeStats() { clusterManagerThrottlingStats, weightedRoutingStats, null, + null, null ); } - private IngestStats.Stats getPipelineStats(List pipelineStats, String id) { + private OperationStats getPipelineStats(List pipelineStats, String id) { return pipelineStats.stream().filter(p1 -> p1.getPipelineId().equals(id)).findFirst().map(p2 -> p2.getStats()).orElse(null); } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index d3a40868bc389..627ada7092273 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -89,15 +89,15 @@ public void testIngestStats() throws Exception { processorStats.compute(stat.getType(), (key, value) -> { if (value == null) { return new long[] { - stat.getStats().getIngestCount(), - stat.getStats().getIngestFailedCount(), - stat.getStats().getIngestCurrent(), - stat.getStats().getIngestTimeInMillis() }; + stat.getStats().getCount(), + stat.getStats().getFailedCount(), + stat.getStats().getCurrent(), + stat.getStats().getTotalTimeInMillis() }; } else { - value[0] += stat.getStats().getIngestCount(); - value[1] += stat.getStats().getIngestFailedCount(); - value[2] += stat.getStats().getIngestCurrent(); - value[3] += stat.getStats().getIngestTimeInMillis(); + value[0] += stat.getStats().getCount(); + value[1] += stat.getStats().getFailedCount(); + value[2] += stat.getStats().getCurrent(); + value[3] += stat.getStats().getTotalTimeInMillis(); return value; } }); diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java index 73349d45bd5c7..e5833ea619774 100644 --- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java @@ -189,6 +189,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -214,6 +215,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -239,6 +241,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ) ); @@ -295,6 +298,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -320,6 +324,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -345,6 +350,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ) ); diff --git a/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java b/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java index b299ac4d66996..76301acac0c19 100644 --- a/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java +++ b/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java @@ -33,6 +33,7 @@ package org.opensearch.ingest; import org.opensearch.OpenSearchException; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -433,10 +434,10 @@ private void assertStats(CompoundProcessor compoundProcessor, long count, long f } private void assertStats(int processor, CompoundProcessor compoundProcessor, long current, long count, long failed, long time) { - IngestStats.Stats stats = compoundProcessor.getProcessorsWithMetrics().get(processor).v2().createStats(); - assertThat(stats.getIngestCount(), equalTo(count)); - assertThat(stats.getIngestCurrent(), equalTo(current)); - assertThat(stats.getIngestFailedCount(), equalTo(failed)); - assertThat(stats.getIngestTimeInMillis(), equalTo(time)); + OperationStats stats = compoundProcessor.getProcessorsWithMetrics().get(processor).v2().createStats(); + assertThat(stats.getCount(), equalTo(count)); + assertThat(stats.getCurrent(), equalTo(current)); + assertThat(stats.getFailedCount(), equalTo(failed)); + assertThat(stats.getTotalTimeInMillis(), equalTo(time)); } } diff --git a/server/src/test/java/org/opensearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/opensearch/ingest/ConditionalProcessorTests.java index a383ab9b97918..921ac10c02862 100644 --- a/server/src/test/java/org/opensearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/opensearch/ingest/ConditionalProcessorTests.java @@ -32,6 +32,7 @@ package org.opensearch.ingest; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; import org.opensearch.script.IngestConditionalScript; import org.opensearch.script.MockScriptEngine; @@ -250,10 +251,10 @@ private static void assertMutatingCtxThrows(Consumer> mutati } private static void assertStats(ConditionalProcessor conditionalProcessor, long count, long failed, long time) { - IngestStats.Stats stats = conditionalProcessor.getMetric().createStats(); - assertThat(stats.getIngestCount(), equalTo(count)); - assertThat(stats.getIngestCurrent(), equalTo(0L)); - assertThat(stats.getIngestFailedCount(), equalTo(failed)); - assertThat(stats.getIngestTimeInMillis(), greaterThanOrEqualTo(time)); + OperationStats stats = conditionalProcessor.getMetric().createStats(); + assertThat(stats.getCount(), equalTo(count)); + assertThat(stats.getCurrent(), equalTo(0L)); + assertThat(stats.getFailedCount(), equalTo(failed)); + assertThat(stats.getTotalTimeInMillis(), greaterThanOrEqualTo(time)); } } diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java index 4176a32e32ad3..19fef468c529e 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java @@ -58,6 +58,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; import org.opensearch.common.bytes.BytesArray; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.xcontent.XContentBuilder; @@ -1739,14 +1740,14 @@ private void assertPipelineStats(List pipelineStats, S assertStats(getPipelineStats(pipelineStats, pipelineId), count, failed, time); } - private void assertStats(IngestStats.Stats stats, long count, long failed, long time) { - assertThat(stats.getIngestCount(), equalTo(count)); - assertThat(stats.getIngestCurrent(), equalTo(0L)); - assertThat(stats.getIngestFailedCount(), equalTo(failed)); - assertThat(stats.getIngestTimeInMillis(), greaterThanOrEqualTo(time)); + private void assertStats(OperationStats stats, long count, long failed, long time) { + assertThat(stats.getCount(), equalTo(count)); + assertThat(stats.getCurrent(), equalTo(0L)); + assertThat(stats.getFailedCount(), equalTo(failed)); + assertThat(stats.getTotalTimeInMillis(), greaterThanOrEqualTo(time)); } - private IngestStats.Stats getPipelineStats(List pipelineStats, String id) { + private OperationStats getPipelineStats(List pipelineStats, String id) { return pipelineStats.stream().filter(p1 -> p1.getPipelineId().equals(id)).findFirst().map(p2 -> p2.getStats()).orElse(null); } } diff --git a/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java b/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java index b5c74f0ee5d16..b17e24ee5424d 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java @@ -35,6 +35,7 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -48,7 +49,7 @@ public class IngestStatsTests extends OpenSearchTestCase { public void testSerialization() throws IOException { - IngestStats.Stats totalStats = new IngestStats.Stats(50, 100, 200, 300); + OperationStats totalStats = new OperationStats(50, 100, 200, 300); List pipelineStats = createPipelineStats(); Map> processorStats = createProcessorStats(pipelineStats); IngestStats ingestStats = new IngestStats(totalStats, pipelineStats, processorStats); @@ -57,20 +58,20 @@ public void testSerialization() throws IOException { } private List createPipelineStats() { - IngestStats.PipelineStat pipeline1Stats = new IngestStats.PipelineStat("pipeline1", new IngestStats.Stats(3, 3, 3, 3)); - IngestStats.PipelineStat pipeline2Stats = new IngestStats.PipelineStat("pipeline2", new IngestStats.Stats(47, 97, 197, 297)); - IngestStats.PipelineStat pipeline3Stats = new IngestStats.PipelineStat("pipeline3", new IngestStats.Stats(0, 0, 0, 0)); + IngestStats.PipelineStat pipeline1Stats = new IngestStats.PipelineStat("pipeline1", new OperationStats(3, 3, 3, 3)); + IngestStats.PipelineStat pipeline2Stats = new IngestStats.PipelineStat("pipeline2", new OperationStats(47, 97, 197, 297)); + IngestStats.PipelineStat pipeline3Stats = new IngestStats.PipelineStat("pipeline3", new OperationStats(0, 0, 0, 0)); return Stream.of(pipeline1Stats, pipeline2Stats, pipeline3Stats).collect(Collectors.toList()); } private Map> createProcessorStats(List pipelineStats) { assert (pipelineStats.size() >= 2); - IngestStats.ProcessorStat processor1Stat = new IngestStats.ProcessorStat("processor1", "type", new IngestStats.Stats(1, 1, 1, 1)); - IngestStats.ProcessorStat processor2Stat = new IngestStats.ProcessorStat("processor2", "type", new IngestStats.Stats(2, 2, 2, 2)); + IngestStats.ProcessorStat processor1Stat = new IngestStats.ProcessorStat("processor1", "type", new OperationStats(1, 1, 1, 1)); + IngestStats.ProcessorStat processor2Stat = new IngestStats.ProcessorStat("processor2", "type", new OperationStats(2, 2, 2, 2)); IngestStats.ProcessorStat processor3Stat = new IngestStats.ProcessorStat( "processor3", "type", - new IngestStats.Stats(47, 97, 197, 297) + new OperationStats(47, 97, 197, 297) ); // pipeline1 -> processor1,processor2; pipeline2 -> processor3 return MapBuilder.>newMapBuilder() @@ -132,14 +133,14 @@ private void assertIngestStats( } - private void assertStats(IngestStats.Stats fromObject, IngestStats.Stats fromStream) { - assertEquals(fromObject.getIngestCount(), fromStream.getIngestCount()); - assertEquals(fromObject.getIngestFailedCount(), fromStream.getIngestFailedCount()); - assertEquals(fromObject.getIngestTimeInMillis(), fromStream.getIngestTimeInMillis()); - assertEquals(fromObject.getIngestCurrent(), fromStream.getIngestCurrent()); + private void assertStats(OperationStats fromObject, OperationStats fromStream) { + assertEquals(fromObject.getCount(), fromStream.getCount()); + assertEquals(fromObject.getFailedCount(), fromStream.getFailedCount()); + assertEquals(fromObject.getTotalTimeInMillis(), fromStream.getTotalTimeInMillis()); + assertEquals(fromObject.getCurrent(), fromStream.getCurrent()); } - private IngestStats.Stats getPipelineStats(List pipelineStats, String id) { + private OperationStats getPipelineStats(List pipelineStats, String id) { return pipelineStats.stream().filter(p1 -> p1.getPipelineId().equals(id)).findFirst().map(p2 -> p2.getStats()).orElse(null); } } diff --git a/server/src/test/java/org/opensearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/opensearch/ingest/PipelineProcessorTests.java index 9f8dda15eeb65..3708b5bc32955 100644 --- a/server/src/test/java/org/opensearch/ingest/PipelineProcessorTests.java +++ b/server/src/test/java/org/opensearch/ingest/PipelineProcessorTests.java @@ -32,6 +32,7 @@ package org.opensearch.ingest; import org.opensearch.OpenSearchException; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.script.ScriptService; import org.opensearch.script.TemplateScript; import org.opensearch.test.OpenSearchTestCase; @@ -192,29 +193,29 @@ public void testPipelineProcessorWithPipelineChain() throws Exception { assertNotNull(ingestDocument.getSourceAndMetadata().get(key1)); // check the stats - IngestStats.Stats pipeline1Stats = pipeline1.getMetrics().createStats(); - IngestStats.Stats pipeline2Stats = pipeline2.getMetrics().createStats(); - IngestStats.Stats pipeline3Stats = pipeline3.getMetrics().createStats(); + OperationStats pipeline1Stats = pipeline1.getMetrics().createStats(); + OperationStats pipeline2Stats = pipeline2.getMetrics().createStats(); + OperationStats pipeline3Stats = pipeline3.getMetrics().createStats(); // current - assertThat(pipeline1Stats.getIngestCurrent(), equalTo(0L)); - assertThat(pipeline2Stats.getIngestCurrent(), equalTo(0L)); - assertThat(pipeline3Stats.getIngestCurrent(), equalTo(0L)); + assertThat(pipeline1Stats.getCurrent(), equalTo(0L)); + assertThat(pipeline2Stats.getCurrent(), equalTo(0L)); + assertThat(pipeline3Stats.getCurrent(), equalTo(0L)); // count - assertThat(pipeline1Stats.getIngestCount(), equalTo(1L)); - assertThat(pipeline2Stats.getIngestCount(), equalTo(1L)); - assertThat(pipeline3Stats.getIngestCount(), equalTo(1L)); + assertThat(pipeline1Stats.getCount(), equalTo(1L)); + assertThat(pipeline2Stats.getCount(), equalTo(1L)); + assertThat(pipeline3Stats.getCount(), equalTo(1L)); // time - assertThat(pipeline1Stats.getIngestTimeInMillis(), equalTo(0L)); - assertThat(pipeline2Stats.getIngestTimeInMillis(), equalTo(3L)); - assertThat(pipeline3Stats.getIngestTimeInMillis(), equalTo(2L)); + assertThat(pipeline1Stats.getTotalTimeInMillis(), equalTo(0L)); + assertThat(pipeline2Stats.getTotalTimeInMillis(), equalTo(3L)); + assertThat(pipeline3Stats.getTotalTimeInMillis(), equalTo(2L)); // failure - assertThat(pipeline1Stats.getIngestFailedCount(), equalTo(0L)); - assertThat(pipeline2Stats.getIngestFailedCount(), equalTo(0L)); - assertThat(pipeline3Stats.getIngestFailedCount(), equalTo(1L)); + assertThat(pipeline1Stats.getFailedCount(), equalTo(0L)); + assertThat(pipeline2Stats.getFailedCount(), equalTo(0L)); + assertThat(pipeline3Stats.getFailedCount(), equalTo(1L)); } public void testIngestPipelineMetadata() { diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index d49d9fd41031c..219dddff40b35 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -30,6 +30,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.xcontent.XContentType; @@ -786,4 +787,126 @@ public void testExceptionOnResponseProcessing() throws Exception { // Exception thrown when processing response expectThrows(SearchPipelineProcessingException.class, () -> pipelinedRequest.transformResponse(response)); } + + public void testStats() throws Exception { + SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", "1", null, r -> { + throw new RuntimeException(); + }); + Map> requestProcessors = Map.of( + "successful_request", + (pf, t, f, c) -> new FakeRequestProcessor("successful_request", "2", null, r -> {}), + "throwing_request", + (pf, t, f, c) -> throwingRequestProcessor + ); + SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", "3", null, r -> { + throw new RuntimeException(); + }); + Map> responseProcessors = Map.of( + "successful_response", + (pf, t, f, c) -> new FakeResponseProcessor("successful_response", "4", null, r -> {}), + "throwing_response", + (pf, t, f, c) -> throwingResponseProcessor + ); + SearchPipelineService searchPipelineService = createWithProcessors(requestProcessors, responseProcessors); + + SearchPipelineMetadata metadata = new SearchPipelineMetadata( + Map.of( + "good_response_pipeline", + new PipelineConfiguration( + "good_response_pipeline", + new BytesArray("{\"response_processors\" : [ { \"successful_response\": {} } ] }"), + XContentType.JSON + ), + "bad_response_pipeline", + new PipelineConfiguration( + "bad_response_pipeline", + new BytesArray("{\"response_processors\" : [ { \"throwing_response\": {} } ] }"), + XContentType.JSON + ), + "good_request_pipeline", + new PipelineConfiguration( + "good_request_pipeline", + new BytesArray("{\"request_processors\" : [ { \"successful_request\": {} } ] }"), + XContentType.JSON + ), + "bad_request_pipeline", + new PipelineConfiguration( + "bad_request_pipeline", + new BytesArray("{\"request_processors\" : [ { \"throwing_request\": {} } ] }"), + XContentType.JSON + ) + ) + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousState = clusterState; + clusterState = ClusterState.builder(clusterState) + .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata)) + .build(); + searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState)); + + SearchRequest request = new SearchRequest(); + SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null); + + searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")).transformResponse(response); + expectThrows( + SearchPipelineProcessingException.class, + () -> searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")).transformResponse(response) + ); + searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")).transformResponse(response); + expectThrows( + SearchPipelineProcessingException.class, + () -> searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")).transformResponse(response) + ); + + SearchPipelineStats stats = searchPipelineService.stats(); + assertPipelineStats(stats.getTotalRequestStats(), 2, 1); + assertPipelineStats(stats.getTotalResponseStats(), 2, 1); + for (SearchPipelineStats.PerPipelineStats perPipelineStats : stats.getPipelineStats()) { + SearchPipelineStats.PipelineDetailStats detailStats = stats.getPerPipelineProcessorStats() + .get(perPipelineStats.getPipelineId()); + switch (perPipelineStats.getPipelineId()) { + case "good_request_pipeline": + assertPipelineStats(perPipelineStats.getRequestStats(), 1, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 0, 0); + assertEquals(1, detailStats.requestProcessorStats().size()); + assertEquals(0, detailStats.responseProcessorStats().size()); + assertEquals("successful_request:2", detailStats.requestProcessorStats().get(0).getProcessorName()); + assertEquals("successful_request", detailStats.requestProcessorStats().get(0).getProcessorType()); + assertPipelineStats(detailStats.requestProcessorStats().get(0).getStats(), 1, 0); + break; + case "bad_request_pipeline": + assertPipelineStats(perPipelineStats.getRequestStats(), 1, 1); + assertPipelineStats(perPipelineStats.getResponseStats(), 0, 0); + assertEquals(1, detailStats.requestProcessorStats().size()); + assertEquals(0, detailStats.responseProcessorStats().size()); + assertEquals("throwing_request:1", detailStats.requestProcessorStats().get(0).getProcessorName()); + assertEquals("throwing_request", detailStats.requestProcessorStats().get(0).getProcessorType()); + assertPipelineStats(detailStats.requestProcessorStats().get(0).getStats(), 1, 1); + break; + case "good_response_pipeline": + assertPipelineStats(perPipelineStats.getRequestStats(), 0, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 1, 0); + assertEquals(0, detailStats.requestProcessorStats().size()); + assertEquals(1, detailStats.responseProcessorStats().size()); + assertEquals("successful_response:4", detailStats.responseProcessorStats().get(0).getProcessorName()); + assertEquals("successful_response", detailStats.responseProcessorStats().get(0).getProcessorType()); + assertPipelineStats(detailStats.responseProcessorStats().get(0).getStats(), 1, 0); + break; + case "bad_response_pipeline": + assertPipelineStats(perPipelineStats.getRequestStats(), 0, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 1, 1); + assertEquals(0, detailStats.requestProcessorStats().size()); + assertEquals(1, detailStats.responseProcessorStats().size()); + assertEquals("throwing_response:3", detailStats.responseProcessorStats().get(0).getProcessorName()); + assertEquals("throwing_response", detailStats.responseProcessorStats().get(0).getProcessorType()); + assertPipelineStats(detailStats.responseProcessorStats().get(0).getStats(), 1, 1); + break; + } + } + } + + private static void assertPipelineStats(OperationStats stats, long count, long failed) { + assertEquals(stats.getCount(), count); + assertEquals(stats.getFailedCount(), failed); + } } diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java new file mode 100644 index 0000000000000..dac41f0db4e00 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline; + +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.metrics.OperationStats; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class SearchPipelineStatsTests extends OpenSearchTestCase { + public void testSerializationRoundtrip() throws IOException { + SearchPipelineStats stats = createStats(); + SearchPipelineStats deserialized; + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + stats.writeTo(bytesStreamOutput); + try (StreamInput bytesStreamInput = bytesStreamOutput.bytes().streamInput()) { + deserialized = new SearchPipelineStats(bytesStreamInput); + } + } + assertEquals(stats, deserialized); + } + + private static SearchPipelineStats createStats() { + return new SearchPipelineStats( + new OperationStats(1, 2, 3, 4), + new OperationStats(5, 6, 7, 8), + List.of( + new SearchPipelineStats.PerPipelineStats("p1", new OperationStats(9, 10, 11, 12), new OperationStats(13, 14, 15, 16)), + new SearchPipelineStats.PerPipelineStats("p2", new OperationStats(17, 18, 19, 20), new OperationStats(21, 22, 23, 24)) + + ), + Map.of( + "p1", + new SearchPipelineStats.PipelineDetailStats( + List.of(new SearchPipelineStats.ProcessorStats("req1:a", "req1", new OperationStats(25, 26, 27, 28))), + List.of(new SearchPipelineStats.ProcessorStats("rsp1:a", "rsp1", new OperationStats(29, 30, 31, 32))) + ), + "p2", + new SearchPipelineStats.PipelineDetailStats( + List.of( + new SearchPipelineStats.ProcessorStats("req1:a", "req1", new OperationStats(33, 34, 35, 36)), + new SearchPipelineStats.ProcessorStats("req2", "req2", new OperationStats(37, 38, 39, 40)) + ), + List.of() + ) + ) + ); + } + + public void testToXContent() throws IOException { + XContentBuilder actualBuilder = XContentBuilder.builder(JsonXContent.jsonXContent); + actualBuilder.startObject(); + createStats().toXContent(actualBuilder, null); + actualBuilder.endObject(); + + String expected = "{" + + " \"search_pipeline\" : {" + + " \"total_request\" : {" + + " \"count\" : 1," + + " \"time_in_millis\" : 2," + + " \"current\" : 3," + + " \"failed\" : 4" + + " }," + + " \"total_response\" : {" + + " \"count\" : 5," + + " \"time_in_millis\" : 6," + + " \"current\" : 7," + + " \"failed\" : 8" + + " }," + + " \"pipelines\" : {" + + " \"p1\" : {" + + " \"request\" : {" + + " \"count\" : 9," + + " \"time_in_millis\" : 10," + + " \"current\" : 11," + + " \"failed\" : 12" + + " }," + + " \"response\" : {" + + " \"count\" : 13," + + " \"time_in_millis\" : 14," + + " \"current\" : 15," + + " \"failed\" : 16" + + " }," + + " \"request_processors\" : [" + + " {" + + " \"req1:a\" : {" + + " \"type\" : \"req1\"," + + " \"stats\" : {" + + " \"count\" : 25," + + " \"time_in_millis\" : 26," + + " \"current\" : 27," + + " \"failed\" : 28" + + " }" + + " }" + + " }" + + " ]," + + " \"response_processors\" : [" + + " {" + + " \"rsp1:a\" : {" + + " \"type\" : \"rsp1\"," + + " \"stats\" : {" + + " \"count\" : 29," + + " \"time_in_millis\" : 30," + + " \"current\" : 31," + + " \"failed\" : 32" + + " }" + + " }" + + " }" + + " ]" + + " }," + + " \"p2\" : {" + + " \"request\" : {" + + " \"count\" : 17," + + " \"time_in_millis\" : 18," + + " \"current\" : 19," + + " \"failed\" : 20" + + " }," + + " \"response\" : {" + + " \"count\" : 21," + + " \"time_in_millis\" : 22," + + " \"current\" : 23," + + " \"failed\" : 24" + + " }," + + " \"request_processors\" : [" + + " {" + + " \"req1:a\" : {" + + " \"type\" : \"req1\"," + + " \"stats\" : {" + + " \"count\" : 33," + + " \"time_in_millis\" : 34," + + " \"current\" : 35," + + " \"failed\" : 36" + + " }" + + " }" + + " }," + + " {" + + " \"req2\" : {" + + " \"type\" : \"req2\"," + + " \"stats\" : {" + + " \"count\" : 37," + + " \"time_in_millis\" : 38," + + " \"current\" : 39," + + " \"failed\" : 40" + + " }" + + " }" + + " }" + + " ]," + + " \"response_processors\" : [ ]" + + " }" + + " }" + + " }" + + "}"; + + XContentParser expectedParser = JsonXContent.jsonXContent.createParser( + this.xContentRegistry(), + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + expected + ); + XContentBuilder expectedBuilder = XContentBuilder.builder(JsonXContent.jsonXContent); + expectedBuilder.generator().copyCurrentStructure(expectedParser); + + assertEquals( + XContentHelper.convertToMap(BytesReference.bytes(expectedBuilder), false, (MediaType) XContentType.JSON), + XContentHelper.convertToMap(BytesReference.bytes(actualBuilder), false, (MediaType) XContentType.JSON) + ); + } +} diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java index cf5f6613c3ea1..6634d1b4dbafc 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java @@ -119,7 +119,8 @@ List adjustNodesStats(List nodesStats) { nodeStats.getClusterManagerThrottlingStats(), nodeStats.getWeightedRoutingStats(), nodeStats.getFileCacheStats(), - nodeStats.getTaskCancellationStats() + nodeStats.getTaskCancellationStats(), + nodeStats.getSearchPipelineStats() ); }).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 38617d09b703d..49d8b64bc71cd 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2685,6 +2685,7 @@ public void ensureEstimatedStats() { false, false, false, + false, false ); assertThat( From b7cace575c84d94b49c02cd8328835d3f8b1a0d0 Mon Sep 17 00:00:00 2001 From: Ketan Verma <9292653+ketanv3@users.noreply.github.com> Date: Thu, 29 Jun 2023 03:44:44 +0530 Subject: [PATCH 021/186] Self-organizing hash table to improve the performance of bucket aggregations (#7652) * Add self-organizing hash table to improve the performance of bucket aggregations Signed-off-by: Ketan Verma * Updated approach: PSL, fingerprint and recency information are embedded in the hash table itself Signed-off-by: Ketan Verma * Updated tests and added microbenchmarks Signed-off-by: Ketan Verma * Renamed FastLongHash to ReorganizingLongHash and updated the default initial capacity Signed-off-by: Ketan Verma --------- Signed-off-by: Ketan Verma --- CHANGELOG.md | 1 + .../common/util/LongHashBenchmark.java | 425 ++++++++++++++++++ .../common/util/ReorganizingLongHash.java | 307 +++++++++++++ .../bucket/terms/LongKeyedBucketOrds.java | 6 +- .../util/ReorganizingLongHashTests.java | 146 ++++++ 5 files changed, 882 insertions(+), 3 deletions(-) create mode 100644 benchmarks/src/main/java/org/opensearch/common/util/LongHashBenchmark.java create mode 100644 server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java create mode 100644 server/src/test/java/org/opensearch/common/util/ReorganizingLongHashTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 109401a46e5b7..5b3b8e251f205 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Compress and cache cluster state during validate join request ([#7321](https://github.com/opensearch-project/OpenSearch/pull/7321)) - [Snapshot Interop] Add Changes in Create Snapshot Flow for remote store interoperability. ([#7118](https://github.com/opensearch-project/OpenSearch/pull/7118)) - Allow insecure string settings to warn-log usage and advise to migration of a newer secure variant ([#5496](https://github.com/opensearch-project/OpenSearch/pull/5496)) +- Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652)) ### Deprecated diff --git a/benchmarks/src/main/java/org/opensearch/common/util/LongHashBenchmark.java b/benchmarks/src/main/java/org/opensearch/common/util/LongHashBenchmark.java new file mode 100644 index 0000000000000..fa75dd2c91f5a --- /dev/null +++ b/benchmarks/src/main/java/org/opensearch/common/util/LongHashBenchmark.java @@ -0,0 +1,425 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.opensearch.common.lease.Releasable; + +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +@Fork(value = 3) +@Warmup(iterations = 1, time = 4) +@Measurement(iterations = 3, time = 2) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public class LongHashBenchmark { + + @Benchmark + public void add(Blackhole bh, HashTableOptions tableOpts, WorkloadOptions workloadOpts) { + try (HashTable table = tableOpts.get(); WorkloadIterator iter = workloadOpts.iter()) { + while (iter.hasNext()) { + bh.consume(table.add(iter.next())); + } + } + } + + /** + * Creates a hash table with varying parameters. + */ + @State(Scope.Benchmark) + public static class HashTableOptions { + + @Param({ "LongHash", "ReorganizingLongHash" }) + public String type; + + @Param({ "1" }) + public long initialCapacity; + + @Param({ "0.6" }) + public float loadFactor; + + private Supplier supplier; + + @Setup + public void setup() { + switch (type) { + case "LongHash": + supplier = this::newLongHash; + break; + case "ReorganizingLongHash": + supplier = this::newReorganizingLongHash; + break; + default: + throw new IllegalArgumentException("invalid hash table type: " + type); + } + } + + public HashTable get() { + return supplier.get(); + } + + private HashTable newLongHash() { + return new HashTable() { + private final LongHash table = new LongHash(initialCapacity, loadFactor, BigArrays.NON_RECYCLING_INSTANCE); + + @Override + public long add(long key) { + return table.add(key); + } + + @Override + public void close() { + table.close(); + } + }; + } + + private HashTable newReorganizingLongHash() { + return new HashTable() { + private final ReorganizingLongHash table = new ReorganizingLongHash( + initialCapacity, + loadFactor, + BigArrays.NON_RECYCLING_INSTANCE + ); + + @Override + public long add(long key) { + return table.add(key); + } + + @Override + public void close() { + table.close(); + } + }; + } + } + + /** + * Creates a workload with varying parameters. + */ + @State(Scope.Benchmark) + public static class WorkloadOptions { + public static final int NUM_HITS = 20_000_000; + + /** + * Repeat the experiment with growing number of keys. + * These values are generated with an exponential growth pattern such that: + * value = ceil(previous_value * random_float_between(1.0, 1.14)) + */ + @Param({ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "13", + "15", + "17", + "18", + "19", + "20", + "21", + "23", + "26", + "27", + "30", + "32", + "35", + "41", + "45", + "50", + "53", + "54", + "55", + "57", + "63", + "64", + "69", + "74", + "80", + "84", + "91", + "98", + "101", + "111", + "114", + "124", + "128", + "139", + "148", + "161", + "162", + "176", + "190", + "204", + "216", + "240", + "257", + "269", + "291", + "302", + "308", + "327", + "341", + "374", + "402", + "412", + "438", + "443", + "488", + "505", + "558", + "612", + "621", + "623", + "627", + "642", + "717", + "765", + "787", + "817", + "915", + "962", + "1011", + "1083", + "1163", + "1237", + "1301", + "1424", + "1541", + "1716", + "1805", + "1817", + "1934", + "2024", + "2238", + "2281", + "2319", + "2527", + "2583", + "2639", + "2662", + "2692", + "2991", + "3201", + "3215", + "3517", + "3681", + "3710", + "4038", + "4060", + "4199", + "4509", + "4855", + "5204", + "5624", + "6217", + "6891", + "7569", + "8169", + "8929", + "9153", + "10005", + "10624", + "10931", + "12070", + "12370", + "13694", + "14227", + "15925", + "17295", + "17376", + "18522", + "19200", + "20108", + "21496", + "23427", + "24224", + "26759", + "29199", + "29897", + "32353", + "33104", + "36523", + "38480", + "38958", + "40020", + "44745", + "45396", + "47916", + "49745", + "49968", + "52231", + "53606" }) + public int size; + + @Param({ "correlated", "uncorrelated", "distinct" }) + public String dataset; + + private WorkloadIterator iterator; + + @Setup + public void setup() { + switch (dataset) { + case "correlated": + iterator = newCorrelatedWorkload(); + break; + case "uncorrelated": + iterator = newUncorrelatedWorkload(); + break; + case "distinct": + iterator = newDistinctWorkload(); + break; + default: + throw new IllegalArgumentException("invalid dataset: " + dataset); + } + } + + public WorkloadIterator iter() { + return iterator; + } + + /** + * Simulates monotonically increasing timestamp data with multiple hits mapping to the same key. + */ + private WorkloadIterator newCorrelatedWorkload() { + assert NUM_HITS >= size : "ensure hits >= size so that each key is used at least once"; + + final long[] data = new long[size]; + for (int i = 0; i < data.length; i++) { + data[i] = 1420070400000L + 3600000L * i; + } + + return new WorkloadIterator() { + private int count = 0; + private int index = 0; + private int remaining = NUM_HITS / data.length; + + @Override + public boolean hasNext() { + return count < NUM_HITS; + } + + @Override + public long next() { + if (--remaining <= 0) { + index = (index + 1) % data.length; + remaining = NUM_HITS / data.length; + } + count++; + return data[index]; + } + + @Override + public void reset() { + count = 0; + index = 0; + remaining = NUM_HITS / data.length; + } + }; + } + + /** + * Simulates uncorrelated data (such as travel distance / fare amount). + */ + private WorkloadIterator newUncorrelatedWorkload() { + assert NUM_HITS >= size : "ensure hits >= size so that each key is used at least once"; + + final Random random = new Random(0); // fixed seed for reproducible results + final long[] data = new long[size]; + for (int i = 0; i < data.length; i++) { + data[i] = Double.doubleToLongBits(20.0 + 80 * random.nextDouble()); + } + + return new WorkloadIterator() { + private int count = 0; + private int index = 0; + + @Override + public boolean hasNext() { + return count < NUM_HITS; + } + + @Override + public long next() { + count++; + index = (index + 1) % data.length; + return data[index]; + } + + @Override + public void reset() { + count = 0; + index = 0; + } + }; + } + + /** + * Simulates workload with high cardinality, i.e., each hit mapping to a different key. + */ + private WorkloadIterator newDistinctWorkload() { + return new WorkloadIterator() { + private int count = 0; + + @Override + public boolean hasNext() { + return count < size; + } + + @Override + public long next() { + return count++; + } + + @Override + public void reset() { + count = 0; + } + }; + } + } + + private interface HashTable extends Releasable { + long add(long key); + } + + private interface WorkloadIterator extends Releasable { + boolean hasNext(); + + long next(); + + void reset(); + + @Override + default void close() { + reset(); + } + } +} diff --git a/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java b/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java new file mode 100644 index 0000000000000..5789b47423c1d --- /dev/null +++ b/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java @@ -0,0 +1,307 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +import org.apache.lucene.util.hppc.BitMixer; +import org.opensearch.common.lease.Releasable; + +/** + * Specialized hash table implementation that maps a (primitive) long to long. + * + *

+ * It organizes itself by moving keys around dynamically in order to reduce the + * longest probe sequence length (PSL), which makes lookups faster as keys are likely to + * be found in the same CPU cache line. It also optimizes lookups for recently added keys, + * making it useful for aggregations where keys are correlated across consecutive hits. + * + *

+ * This class is not thread-safe. + * + * @opensearch.internal + */ +public class ReorganizingLongHash implements Releasable { + private static final long MAX_CAPACITY = 1L << 32; + private static final long DEFAULT_INITIAL_CAPACITY = 32; + private static final float DEFAULT_LOAD_FACTOR = 0.6f; + + /** + * Maximum load factor after which the capacity is doubled. + */ + private final float loadFactor; + + /** + * Utility class to allocate recyclable arrays. + */ + private final BigArrays bigArrays; + + /** + * Current capacity of the hash table. This must be a power of two so that the hash table slot + * can be identified quickly using bitmasks, thus avoiding expensive modulo or integer division. + */ + private long capacity; + + /** + * Bitmask to identify the hash table slot from a key's hash. + */ + private long mask; + + /** + * Size threshold after which the hash table needs to be doubled in capacity. + */ + private long grow; + + /** + * Current size of the hash table. + */ + private long size; + + /** + * Underlying array to store the hash table values. + * + *

+ * Each hash table value (64-bit) uses the following byte packing strategy: + * 

+     * |=========|===============|================|================================|
+     * | Discard | PSL           | Fingerprint    | Ordinal                        |
+     * |    -    |---------------|----------------|--------------------------------|
+     * | 1 bit   | 15 bits       | 16 bits        | 32 bits                        |
+     * |=========|===============|================|================================|
+     *
+ * + *

+ * This allows us to encode and manipulate additional information in the hash table + * itself without having to look elsewhere in the memory, which is much slower. + * + *

+ * Terminology: table[index] = value = (discard | psl | fingerprint | ordinal) + */ + private LongArray table; + + /** + * Underlying array to store the keys. + * + *

+ * Terminology: keys[ordinal] = key + */ + private LongArray keys; + + /** + * Bitmasks to manipulate the hash table values. + */ + private static final long MASK_ORDINAL = 0x00000000FFFFFFFFL; // extract ordinal + private static final long MASK_FINGERPRINT = 0x0000FFFF00000000L; // extract fingerprint + private static final long MASK_PSL = 0x7FFF000000000000L; // extract PSL + private static final long INCR_PSL = 0x0001000000000000L; // increment PSL by one + + public ReorganizingLongHash(final BigArrays bigArrays) { + this(DEFAULT_INITIAL_CAPACITY, DEFAULT_LOAD_FACTOR, bigArrays); + } + + public ReorganizingLongHash(final long initialCapacity, final float loadFactor, final BigArrays bigArrays) { + assert initialCapacity > 0 : "initial capacity must be greater than 0"; + assert loadFactor > 0 && loadFactor < 1 : "load factor must be between 0 and 1"; + + this.bigArrays = bigArrays; + this.loadFactor = loadFactor; + + capacity = nextPowerOfTwo((long) (initialCapacity / loadFactor)); + mask = capacity - 1; + grow = (long) (capacity * loadFactor); + size = 0; + + table = bigArrays.newLongArray(capacity, false); + table.fill(0, capacity, -1); // -1 represents an empty slot + keys = bigArrays.newLongArray(initialCapacity, false); + } + + /** + * Adds the given key to the hash table and returns its ordinal. + * If the key exists already, it returns (-1 - ordinal). + */ + public long add(final long key) { + final long ordinal = find(key); + if (ordinal != -1) { + return -1 - ordinal; + } + + if (size >= grow) { + grow(); + } + + return insert(key); + } + + /** + * Returns the key associated with the given ordinal. + * The result is undefined for an unused ordinal. + */ + public long get(final long ordinal) { + return keys.get(ordinal); + } + + /** + * Returns the ordinal associated with the given key, or -1 if the key doesn't exist. + * + *

+ * Using the 64-bit hash value, up to 32 least significant bits (LSB) are used to identify the + * home slot in the hash table, and an additional 16 bits are used to identify the fingerprint. + * The fingerprint further increases the entropy and reduces the number of false lookups in the + * keys' table during equality checks, which is expensive due to an uncorrelated memory lookup. + * + *

+ * Total entropy bits = 16 + log2(capacity) + * + *

+ * Linear probing starts from the home slot, until a match or an empty slot is found. + * Values are first checked using their fingerprint (to reduce false positives), then verified + * in the keys' table using an equality check. + */ + public long find(final long key) { + final long hash = hash(key); + final long fingerprint = hash & MASK_FINGERPRINT; + + for (long idx = hash & mask, value, ordinal;; idx = (idx + 1) & mask) { + if ((value = table.get(idx)) == -1) { + return -1; + } else if (((value & MASK_FINGERPRINT) == fingerprint) && (keys.get((ordinal = (value & MASK_ORDINAL))) == key)) { + return ordinal; + } + } + } + + /** + * Returns the number of mappings in this hash table. + */ + public long size() { + return size; + } + + /** + * Inserts the given key in the hash table and returns its ordinal. + * + *

+ * Inspired by Robin Hood Hashing (RHH): if the PSL for the existing value is less than the PSL + * for the value being inserted, swap the two values and keep going. Values that were inserted + * early and thus "lucked out" on their PSLs will gradually be moved away from their preferred + * slot as new values come in that could make better use of that place in the table. It evens out + * the PSLs across the board and reduces the longest PSL dramatically. + * + *

+ * A lower variance is better because, with modern CPU architectures, a PSL of 1 isn't much + * faster than a PSL of 3, because the main cost is fetching the cache line. The ideal hash + * table layout is the one where all values have equal PSLs, and that PSL fits within one cache line. + * + *

+ * The expected longest PSL for a full table: log(N) + * + *

+ * Our implementation has a slight variation on top of it: by loosening the guarantees provided + * by RHH, we can improve the performance on correlated lookups (such as aggregating on repeated + * timestamps) by moving the "recent" keys closer to their home slot, and eventually converging + * to the ideal hash table layout defined by RHH. + */ + private long insert(final long key) { + final long hash = hash(key); + final long fingerprint = hash & MASK_FINGERPRINT; + + // The ideal home slot for the given key. + long idx = hash & mask; + + // The value yet to find an empty slot (candidate). + long value = fingerprint | size; + + // The existing value at idx. + long existingValue; + + // Always set the newly inserted key at its ideal home slot, even if it doesn't conform + // to the RHH scheme (yet). This will ensure subsequent correlated lookups are fast due + // to no additional probing. When another insertion causes this value to be displaced, it + // will eventually be placed at an appropriate location defined by the RHH scheme. + if ((value = table.set(idx, value)) == -1) { + // The ideal home slot was already empty; append the key and return early. + return append(key); + } + + // Find an alternative slot for the displaced value such that the longest PSL is minimized. + do { + idx = (idx + 1) & mask; + value += INCR_PSL; + + if ((existingValue = table.get(idx)) == -1) { + // Empty slot; insert the candidate value here. + table.set(idx, value); + return append(key); + } else if ((existingValue & MASK_PSL) <= (value & MASK_PSL)) { + // Existing value is "richer" than the candidate value at this index; + // swap and find an alternative slot for the displaced value. + // In the case of a tie, the candidate value (i.e. the recent value) is chosen as + // the winner and kept closer to its ideal home slot in order to speed up + // correlated lookups. + value = table.set(idx, value); + } + } while (true); + } + + /** + * Appends the key in the keys' table. + */ + private long append(final long key) { + keys = bigArrays.grow(keys, size + 1); + keys.set(size, key); + return size++; + } + + /** + * Returns the hash for the given key. + * Visible for unit-tests. + */ + long hash(final long key) { + return BitMixer.mix64(key); + } + + /** + * Returns the underlying hash table. + * Visible for unit-tests. + */ + LongArray getTable() { + return table; + } + + /** + * Grows the hash table by doubling its capacity and reinserting the keys. + */ + private void grow() { + // Ensure that the hash table doesn't grow too large. + // This implicitly also ensures that the ordinals are no larger than 2^32, thus, + // preventing them from polluting other bits (PSL/fingerprint) in the hash table values. + assert capacity < MAX_CAPACITY : "hash table already at the max capacity"; + + final long oldSize = size; + capacity <<= 1; + mask = capacity - 1; + size = 0; + grow = (long) (capacity * loadFactor); + table = bigArrays.resize(table, capacity); + table.fill(0, capacity, -1); + + for (long ordinal = 0; ordinal < oldSize; ordinal++) { + insert(keys.get(ordinal)); + } + } + + @Override + public void close() { + table.close(); + keys.close(); + } + + private static long nextPowerOfTwo(final long value) { + return Math.max(1, Long.highestOneBit(value - 1) << 1); + } +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrds.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrds.java index 6b511587cc271..a55b7d6bc154e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrds.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrds.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.bucket.terms; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.LongHash; +import org.opensearch.common.util.ReorganizingLongHash; import org.opensearch.common.util.LongLongHash; import org.opensearch.common.lease.Releasable; import org.opensearch.search.aggregations.CardinalityUpperBound; @@ -148,10 +148,10 @@ public long value() { * @opensearch.internal */ public static class FromSingle extends LongKeyedBucketOrds { - private final LongHash ords; + private final ReorganizingLongHash ords; public FromSingle(BigArrays bigArrays) { - ords = new LongHash(1, bigArrays); + ords = new ReorganizingLongHash(bigArrays); } @Override diff --git a/server/src/test/java/org/opensearch/common/util/ReorganizingLongHashTests.java b/server/src/test/java/org/opensearch/common/util/ReorganizingLongHashTests.java new file mode 100644 index 0000000000000..259eab6c624bd --- /dev/null +++ b/server/src/test/java/org/opensearch/common/util/ReorganizingLongHashTests.java @@ -0,0 +1,146 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class ReorganizingLongHashTests extends OpenSearchTestCase { + + public void testFuzzy() { + Map reference = new HashMap<>(); + + try ( + ReorganizingLongHash h = new ReorganizingLongHash( + randomIntBetween(1, 100), // random capacity + 0.6f + randomFloat() * 0.39f, // random load factor to verify collision resolution + BigArrays.NON_RECYCLING_INSTANCE + ) + ) { + // Verify the behaviour of "add" and "find". + for (int i = 0; i < (1 << 20); i++) { + long key = randomLong() % (1 << 12); // roughly ~4% unique keys + if (reference.containsKey(key)) { + long expectedOrdinal = reference.get(key); + assertEquals(-1 - expectedOrdinal, h.add(key)); + assertEquals(expectedOrdinal, h.find(key)); + } else { + assertEquals(-1, h.find(key)); + reference.put(key, (long) reference.size()); + assertEquals((long) reference.get(key), h.add(key)); + } + } + + // Verify the behaviour of "get". + for (Map.Entry entry : reference.entrySet()) { + assertEquals((long) entry.getKey(), h.get(entry.getValue())); + } + + // Verify the behaviour of "size". + assertEquals(reference.size(), h.size()); + + // Verify the calculation of PSLs. + final long capacity = h.getTable().size(); + final long mask = capacity - 1; + for (long idx = 0; idx < h.getTable().size(); idx++) { + final long value = h.getTable().get(idx); + if (value != -1) { + final long homeIdx = h.hash(h.get((int) value)) & mask; + assertEquals((capacity + idx - homeIdx) & mask, value >>> 48); + } + } + } + } + + public void testRearrangement() { + try (ReorganizingLongHash h = new ReorganizingLongHash(4, 0.6f, BigArrays.NON_RECYCLING_INSTANCE) { + /** + * Overriding with an "identity" hash function to make it easier to reason about the placement + * of values in the hash table. The backing array of the hash table will have a size (8), + * i.e. nextPowerOfTwo(initialCapacity/loadFactor), so the bitmask will be (7). + * The ideal home slot of a key can then be defined as: (hash(key) & mask) = (key & 7). + */ + @Override + long hash(long key) { + return key; + } + }) { + /* + * Add key=0, hash=0, home_slot=0 + * + * Before: empty slot. + * â–¼ + * [ _ _ _ _ _ _ _ _ ] + * + * After: inserted [ordinal=0, psl=0] at the empty slot. + * [ 0 _ _ _ _ _ _ _ ] + */ + h.add(0); + assertEquals(encodeValue(0, 0, 0), h.getTable().get(0)); + + /* + * Add key=8, hash=8, home_slot=0 + * + * Before: occupied slot. + * â–¼ + * [ 0 _ _ _ _ _ _ _ ] + * + * After: inserted [ordinal=1, psl=0] at the existing slot, displaced [ordinal=0, psl=0], + * and re-inserted it at the next empty slot as [ordinal=0, psl=1]. + * [ 1 0 _ _ _ _ _ _ ] + */ + h.add(8); + assertEquals(encodeValue(0, 0, 1), h.getTable().get(0)); + assertEquals(encodeValue(1, 0, 0), h.getTable().get(1)); + + /* + * Add key=1, hash=1, home_slot=1 + * + * Before: occupied slot. + * â–¼ + * [ 1 0 _ _ _ _ _ _ ] + * + * After: inserted [ordinal=2, psl=0] at the existing slot, displaced [ordinal=0, psl=1], + * and re-inserted it at the next empty slot as [ordinal=0, psl=2]. + * [ 1 2 0 _ _ _ _ _ ] + */ + h.add(1); + assertEquals(encodeValue(0, 0, 1), h.getTable().get(0)); + assertEquals(encodeValue(0, 0, 2), h.getTable().get(1)); + assertEquals(encodeValue(2, 0, 0), h.getTable().get(2)); + + /* + * Add key=16, hash=16, home_slot=0 + * + * Before: occupied slot. + * â–¼ + * [ 1 2 0 _ _ _ _ _ ] + * + * After: inserted [ordinal=3, psl=0] at the existing slot, displaced [ordinal=1, psl=0] + * and re-inserted it at the next best slot. Repeated this for other displaced values + * until everything found an empty slot. + * [ 3 1 0 2 _ _ _ _ ] + */ + h.add(16); + assertEquals(encodeValue(0, 0, 3), h.getTable().get(0)); + assertEquals(encodeValue(1, 0, 1), h.getTable().get(1)); + assertEquals(encodeValue(2, 0, 0), h.getTable().get(2)); + assertEquals(encodeValue(2, 0, 2), h.getTable().get(3)); + } + } + + private static long encodeValue(long psl, long fingerprint, long ordinal) { + assert psl < (1L << 15); + assert fingerprint < (1L << 16); + assert ordinal < (1L << 32); + return (psl << 48) | (fingerprint << 32) | ordinal; + } +} From 74788c3f45895a90d030b2434539beac7a1b1723 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 29 Jun 2023 10:40:55 +1200 Subject: [PATCH 022/186] Bump com.google.api-client:google-api-client from 1.34.0 to 2.2.0 in /plugins/repository-gcs (#8276) * Bump com.google.api-client:google-api-client from 1.34.0 to 2.2.0 in /plugins/repository-gcs Signed-off-by: Thomas Farr * Bump google-http-client and google-oauth-client Signed-off-by: Thomas --------- Signed-off-by: Thomas Farr Signed-off-by: Thomas --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 44 +++++++++++-------- .../google-api-client-1.34.0.jar.sha1 | 1 - .../licenses/google-api-client-2.2.0.jar.sha1 | 1 + ...ces-storage-v1-rev20220608-1.32.1.jar.sha1 | 1 - ...ices-storage-v1-rev20230617-2.0.0.jar.sha1 | 1 + .../google-http-client-gson-1.41.4.jar.sha1 | 1 - .../google-http-client-gson-1.43.2.jar.sha1 | 1 + ...oogle-http-client-jackson2-1.42.3.jar.sha1 | 1 - ...oogle-http-client-jackson2-1.43.2.jar.sha1 | 1 + .../google-oauth-client-1.33.3.jar.sha1 | 1 - .../google-oauth-client-1.34.1.jar.sha1 | 1 + 12 files changed, 32 insertions(+), 23 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-api-client-1.34.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20220608-1.32.1.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-gson-1.41.4.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.42.3.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/google-oauth-client-1.33.3.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3b8e251f205..5ec159efafc9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -122,6 +122,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256) - [Upgrade] Lucene 9.7.0 release (#8272) - Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209)) +- Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276)) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 5d7a4c64ae34a..41c36dffea296 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -52,33 +52,43 @@ versions << [ ] dependencies { - api 'com.google.cloud:google-cloud-storage:1.113.1' + api 'com.google.api:api-common:1.8.1' + api 'com.google.api:gax:2.27.0' + api 'com.google.api:gax-httpjson:0.103.1' + + api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0' + + api 'com.google.api-client:google-api-client:2.2.0' + + api 'com.google.api.grpc:proto-google-common-protos:2.10.0' + api 'com.google.api.grpc:proto-google-iam-v1:0.12.0' + + api "com.google.auth:google-auth-library-credentials:${versions.google_auth}" + api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" + api 'com.google.cloud:google-cloud-core:2.5.10' + api 'com.google.cloud:google-cloud-core-http:2.17.0' + api 'com.google.cloud:google-cloud-storage:1.113.1' + + api 'com.google.code.gson:gson:2.9.0' + runtimeOnly "com.google.guava:guava:${versions.guava}" api 'com.google.guava:failureaccess:1.0.1' + api 'com.google.http-client:google-http-client:1.43.2' + api 'com.google.http-client:google-http-client-appengine:1.43.2' + api 'com.google.http-client:google-http-client-gson:1.43.2' + api 'com.google.http-client:google-http-client-jackson2:1.43.2' + + api 'com.google.oauth-client:google-oauth-client:1.34.1' + api "commons-logging:commons-logging:${versions.commonslogging}" api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" api "commons-codec:commons-codec:${versions.commonscodec}" - api 'com.google.api:api-common:1.8.1' - api 'com.google.api:gax:2.27.0' api 'org.threeten:threetenbp:1.4.4' - api 'com.google.code.gson:gson:2.9.0' - api 'com.google.api.grpc:proto-google-common-protos:2.10.0' - api 'com.google.api.grpc:proto-google-iam-v1:0.12.0' - api 'com.google.cloud:google-cloud-core-http:2.17.0' - api "com.google.auth:google-auth-library-credentials:${versions.google_auth}" - api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" - api 'com.google.oauth-client:google-oauth-client:1.33.3' - api 'com.google.api-client:google-api-client:1.34.0' - api 'com.google.http-client:google-http-client-appengine:1.43.2' - api 'com.google.http-client:google-http-client-jackson2:1.42.3' - api 'com.google.http-client:google-http-client-gson:1.41.4' - api 'com.google.api:gax-httpjson:0.103.1' api 'io.grpc:grpc-context:1.46.0' api 'io.opencensus:opencensus-api:0.31.1' api 'io.opencensus:opencensus-contrib-http-util:0.31.1' - api 'com.google.apis:google-api-services-storage:v1-rev20220608-1.32.1' testImplementation project(':test:fixtures:gcs-fixture') } @@ -167,8 +177,6 @@ thirdPartyAudit { 'org.apache.http.client.methods.HttpRequestBase', 'org.apache.http.config.Registry', 'org.apache.http.config.RegistryBuilder', - 'org.apache.http.config.SocketConfig', - 'org.apache.http.config.SocketConfig$Builder', 'org.apache.http.conn.ClientConnectionManager', 'org.apache.http.conn.ConnectionKeepAliveStrategy', 'org.apache.http.conn.params.ConnManagerParams', diff --git a/plugins/repository-gcs/licenses/google-api-client-1.34.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-1.34.0.jar.sha1 deleted file mode 100644 index 9be9480435085..0000000000000 --- a/plugins/repository-gcs/licenses/google-api-client-1.34.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af2586412cabeee49c9db6d736e75b745bc467f8 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 new file mode 100644 index 0000000000000..f9604d6837ca9 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 @@ -0,0 +1 @@ +10e53fd4d987e37190432e896bdaa62e8ea2c628 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20220608-1.32.1.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20220608-1.32.1.jar.sha1 deleted file mode 100644 index 07aaadb2664b2..0000000000000 --- a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20220608-1.32.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -74724addc6cecac408dad3a6a26423b7647b3724 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..1a1452f773b96 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1 @@ -0,0 +1 @@ +fc3f225b405303fe7cb760d578348b6b07e7ea8b \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.41.4.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.41.4.jar.sha1 deleted file mode 100644 index 17960a99abea2..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-gson-1.41.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa665c1c573765dd858bc34931ad747e4ed11efe \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 new file mode 100644 index 0000000000000..df0374aa27c70 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 @@ -0,0 +1 @@ +b1c2e3e89804c113dba7b342aa8e0fc2cf3d9378 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.42.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.42.3.jar.sha1 deleted file mode 100644 index 34d7d49f7b147..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.42.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -789cafde696403b429026bf19071caf46d8c8934 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 new file mode 100644 index 0000000000000..7b606a07651ed --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 @@ -0,0 +1 @@ +5e52a9967ebd8246fc4cca64df5f03608db5ac6e \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.33.3.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.33.3.jar.sha1 deleted file mode 100644 index f2afaa1bc2dba..0000000000000 --- a/plugins/repository-gcs/licenses/google-oauth-client-1.33.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d445a8649b0de731922b9a3ebf1552b5403611d \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 new file mode 100644 index 0000000000000..a8434bd380761 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1 @@ -0,0 +1 @@ +4a4f88c5e13143f882268c98239fb85c3b2c6cb2 \ No newline at end of file From 976048d5d630b7adc53f54f2b27558d3a38b1f65 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Thu, 29 Jun 2023 06:23:14 +0530 Subject: [PATCH 023/186] Moving zstd out of sandbox (#7908) * Adding zstd module to source Signed-off-by: Sarthak Aggarwal * Removing zstd module from sandbox Signed-off-by: Sarthak Aggarwal * Added tests and refactoring Signed-off-by: Sarthak Aggarwal * Fixing gradle issues Signed-off-by: Sarthak Aggarwal * flaky test Signed-off-by: Sarthak Aggarwal * fixing precommit failure Signed-off-by: Sarthak Aggarwal * Incorporate review comments and fixed precommit failures Signed-off-by: Sarthak Aggarwal * Incorporating review comments Signed-off-by: Sarthak Aggarwal * Incorporating review comments Signed-off-by: Sarthak Aggarwal * Adding Integ tests Signed-off-by: Sarthak Aggarwal * Incorporating review comments Signed-off-by: Sarthak Aggarwal --------- Signed-off-by: Sarthak Aggarwal Signed-off-by: Daniel (dB.) Doubrovkine Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + .../index/codec/MultiCodecReindexIT.java | 189 ++++++++++++++++++ sandbox/plugins/custom-codecs/build.gradle | 28 --- .../codec/customcodecs/CustomCodecPlugin.java | 26 --- .../plugin-metadata/plugin-security.policy | 11 - .../index/codec/MultiCodecMergeIT.java | 175 ++++++++++++++++ .../opensearch/index/codec/CodecService.java | 8 + .../customcodecs/Lucene95CustomCodec.java | 27 ++- .../Lucene95CustomStoredFieldsFormat.java | 13 +- .../index/codec/customcodecs/ZstdCodec.java | 7 + .../customcodecs/ZstdCompressionMode.java | 5 +- .../codec/customcodecs/ZstdNoDictCodec.java | 9 +- .../ZstdNoDictCompressionMode.java | 5 +- .../codec/customcodecs/package-info.java | 0 .../opensearch/index/engine/EngineConfig.java | 4 +- .../services/org.apache.lucene.codecs.Codec | 0 .../opensearch/index/codec/CodecTests.java | 81 ++++++-- .../customcodecs/AbstractCompressorTests.java | 10 +- ...Lucene95CustomStoredFieldsFormatTests.java | 27 +++ .../customcodecs/ZstdCompressorTests.java | 0 .../ZstdNoDictCompressorTests.java | 0 21 files changed, 524 insertions(+), 102 deletions(-) create mode 100644 modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java delete mode 100644 sandbox/plugins/custom-codecs/build.gradle delete mode 100644 sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java delete mode 100644 sandbox/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy create mode 100644 server/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java (65%) rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java (96%) rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java (79%) rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java (99%) rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java (74%) rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java (99%) rename {sandbox/plugins/custom-codecs => server}/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java (100%) rename {sandbox/plugins/custom-codecs => server}/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec (100%) rename {sandbox/plugins/custom-codecs => server}/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java (100%) create mode 100644 server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java rename {sandbox/plugins/custom-codecs => server}/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java (100%) rename {sandbox/plugins/custom-codecs => server}/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec159efafc9a..139807f0bffd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Snapshot Interop] Add Changes in Create Snapshot Flow for remote store interoperability. ([#7118](https://github.com/opensearch-project/OpenSearch/pull/7118)) - Allow insecure string settings to warn-log usage and advise to migration of a newer secure variant ([#5496](https://github.com/opensearch-project/OpenSearch/pull/5496)) - Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652)) +- Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) ### Deprecated diff --git a/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java new file mode 100644 index 0000000000000..87f3c68d8af76 --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec; + +import org.opensearch.action.admin.indices.flush.FlushResponse; +import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.action.support.ActiveShardCount; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.engine.Segment; +import org.opensearch.index.reindex.BulkByScrollResponse; +import org.opensearch.index.reindex.ReindexAction; +import org.opensearch.index.reindex.ReindexRequestBuilder; +import org.opensearch.index.reindex.ReindexTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; + +public class MultiCodecReindexIT extends ReindexTestCase { + + public void testReindexingMultipleCodecs() throws InterruptedException, ExecutionException { + internalCluster().ensureAtLeastNumDataNodes(1); + Map codecMap = Map.of( + "best_compression", + "BEST_COMPRESSION", + "zstd_no_dict", + "ZSTD_NO_DICT", + "zstd", + "ZSTD", + "default", + "BEST_SPEED" + ); + + for (Map.Entry codec : codecMap.entrySet()) { + assertReindexingWithMultipleCodecs(codec.getKey(), codec.getValue(), codecMap); + } + + } + + private void assertReindexingWithMultipleCodecs(String destCodec, String destCodecMode, Map codecMap) + throws ExecutionException, InterruptedException { + + final String index = "test-index" + destCodec; + final String destIndex = "dest-index" + destCodec; + + // creating source index + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", "default") + .put("index.merge.policy.max_merged_segment", "1b") + .build() + ); + ensureGreen(index); + + final int nbDocs = randomIntBetween(2, 5); + + // indexing with all 4 codecs + for (Map.Entry codec : codecMap.entrySet()) { + useCodec(index, codec.getKey()); + ingestDocs(index, nbDocs); + } + + assertTrue( + getSegments(index).stream() + .flatMap(s -> s.getAttributes().values().stream()) + .collect(Collectors.toSet()) + .containsAll(codecMap.values()) + ); + + // creating destination index with destination codec + createIndex( + destIndex, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", destCodec) + .build() + ); + + BulkByScrollResponse bulkResponse = new ReindexRequestBuilder(client(), ReindexAction.INSTANCE).source(index) + .destination(destIndex) + .refresh(true) + .waitForActiveShards(ActiveShardCount.ONE) + .get(); + + assertEquals(codecMap.size() * nbDocs, bulkResponse.getCreated()); + assertEquals(codecMap.size() * nbDocs, bulkResponse.getTotal()); + assertEquals(0, bulkResponse.getDeleted()); + assertEquals(0, bulkResponse.getNoops()); + assertEquals(0, bulkResponse.getVersionConflicts()); + assertEquals(1, bulkResponse.getBatches()); + assertTrue(bulkResponse.getTook().getMillis() > 0); + assertEquals(0, bulkResponse.getBulkFailures().size()); + assertEquals(0, bulkResponse.getSearchFailures().size()); + assertTrue(getSegments(destIndex).stream().allMatch(segment -> segment.attributes.containsValue(destCodecMode))); + } + + private void useCodec(String index, String codec) throws ExecutionException, InterruptedException { + assertAcked(client().admin().indices().prepareClose(index)); + + assertAcked( + client().admin() + .indices() + .updateSettings(new UpdateSettingsRequest(index).settings(Settings.builder().put("index.codec", codec))) + .get() + ); + + assertAcked(client().admin().indices().prepareOpen(index)); + } + + private void flushAndRefreshIndex(String index) { + + // Request is not blocked + for (String blockSetting : Arrays.asList( + SETTING_BLOCKS_READ, + SETTING_BLOCKS_WRITE, + SETTING_READ_ONLY, + SETTING_BLOCKS_METADATA, + SETTING_READ_ONLY_ALLOW_DELETE + )) { + try { + enableIndexBlock(index, blockSetting); + // flush + FlushResponse flushResponse = client().admin().indices().prepareFlush(index).setForce(true).execute().actionGet(); + assertNoFailures(flushResponse); + + // refresh + RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(index).execute().actionGet(); + assertNoFailures(refreshResponse); + } finally { + disableIndexBlock(index, blockSetting); + } + } + } + + private void ingestDocs(String index, int nbDocs) throws InterruptedException { + + indexRandom( + randomBoolean(), + false, + randomBoolean(), + IntStream.range(0, nbDocs) + .mapToObj(i -> client().prepareIndex(index).setId(UUID.randomUUID().toString()).setSource("num", i)) + .collect(toList()) + ); + flushAndRefreshIndex(index); + } + + private ArrayList getSegments(String index) { + + return new ArrayList<>( + client().admin() + .indices() + .segments(new IndicesSegmentsRequest(index)) + .actionGet() + .getIndices() + .get(index) + .getShards() + .get(0) + .getShards()[0].getSegments() + ); + } + +} diff --git a/sandbox/plugins/custom-codecs/build.gradle b/sandbox/plugins/custom-codecs/build.gradle deleted file mode 100644 index 2183df25044a4..0000000000000 --- a/sandbox/plugins/custom-codecs/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' - -opensearchplugin { - name 'custom-codecs' - description 'A plugin that implements custom compression codecs.' - classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' - licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') -} - -dependencies { - api "com.github.luben:zstd-jni:${versions.zstd}" -} - -yamlRestTest.enabled = false; -testingConventions.enabled = false; diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java deleted file mode 100644 index 9d36184bf81af..0000000000000 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.EnginePlugin; - -/** - * A plugin that implements custom codecs. Supports these codecs: - *

    - *
  • zstd - *
  • zstdnodict - *
- * - * @opensearch.internal - */ -public final class CustomCodecPlugin extends Plugin implements EnginePlugin { - /** Creates a new instance. */ - public CustomCodecPlugin() {} -} diff --git a/sandbox/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy b/sandbox/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy deleted file mode 100644 index 8161010cfa897..0000000000000 --- a/sandbox/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy +++ /dev/null @@ -1,11 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -grant codeBase "${codebase.zstd-jni}" { - permission java.lang.RuntimePermission "loadLibrary.*"; -}; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java new file mode 100644 index 0000000000000..2866292e5e2e0 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java @@ -0,0 +1,175 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec; + +import org.opensearch.action.admin.indices.flush.FlushResponse; +import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; +import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.engine.Segment; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.is; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) +public class MultiCodecMergeIT extends OpenSearchIntegTestCase { + + public void testForceMergeMultipleCodecs() throws ExecutionException, InterruptedException { + + Map codecMap = Map.of( + "best_compression", + "BEST_COMPRESSION", + "zstd_no_dict", + "ZSTD_NO_DICT", + "zstd", + "ZSTD", + "default", + "BEST_SPEED" + ); + + for (Map.Entry codec : codecMap.entrySet()) { + forceMergeMultipleCodecs(codec.getKey(), codec.getValue(), codecMap); + } + + } + + private void forceMergeMultipleCodecs(String finalCodec, String finalCodecMode, Map codecMap) throws ExecutionException, + InterruptedException { + + internalCluster().ensureAtLeastNumDataNodes(1); + final String index = "test-index" + finalCodec; + + // creating index + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", "default") + .put("index.merge.policy.max_merged_segment", "1b") + .build() + ); + ensureGreen(index); + // ingesting and asserting segment codec mode for all four codecs + for (Map.Entry codec : codecMap.entrySet()) { + useCodec(index, codec.getKey()); + ingestDocs(index); + } + + assertTrue( + getSegments(index).stream() + .flatMap(s -> s.getAttributes().values().stream()) + .collect(Collectors.toSet()) + .containsAll(codecMap.values()) + ); + + // force merge into final codec + useCodec(index, finalCodec); + flushAndRefreshIndex(index); + final ForceMergeResponse forceMergeResponse = client().admin().indices().prepareForceMerge(index).setMaxNumSegments(1).get(); + + assertThat(forceMergeResponse.getFailedShards(), is(0)); + assertThat(forceMergeResponse.getSuccessfulShards(), is(1)); + + flushAndRefreshIndex(index); + + List segments = getSegments(index).stream().filter(Segment::isSearch).collect(Collectors.toList()); + assertEquals(1, segments.size()); + assertTrue(segments.stream().findFirst().get().attributes.containsValue(finalCodecMode)); + } + + private void useCodec(String index, String codec) throws ExecutionException, InterruptedException { + assertAcked(client().admin().indices().prepareClose(index)); + + assertAcked( + client().admin() + .indices() + .updateSettings(new UpdateSettingsRequest(index).settings(Settings.builder().put("index.codec", codec))) + .get() + ); + + assertAcked(client().admin().indices().prepareOpen(index)); + } + + private void ingestDocs(String index) throws InterruptedException { + ingest(index); + flushAndRefreshIndex(index); + } + + private ArrayList getSegments(String index) { + + return new ArrayList<>( + client().admin() + .indices() + .segments(new IndicesSegmentsRequest(index)) + .actionGet() + .getIndices() + .get(index) + .getShards() + .get(0) + .getShards()[0].getSegments() + ); + } + + private void ingest(String index) throws InterruptedException { + + final int nbDocs = randomIntBetween(1, 5); + indexRandom( + randomBoolean(), + false, + randomBoolean(), + IntStream.range(0, nbDocs) + .mapToObj(i -> client().prepareIndex(index).setId(UUID.randomUUID().toString()).setSource("num", i)) + .collect(toList()) + ); + } + + private void flushAndRefreshIndex(String index) { + + // Request is not blocked + for (String blockSetting : Arrays.asList( + SETTING_BLOCKS_READ, + SETTING_BLOCKS_WRITE, + SETTING_READ_ONLY, + SETTING_BLOCKS_METADATA, + SETTING_READ_ONLY_ALLOW_DELETE + )) { + try { + enableIndexBlock(index, blockSetting); + FlushResponse flushResponse = client().admin().indices().prepareFlush(index).setForce(true).execute().actionGet(); + assertNoFailures(flushResponse); + RefreshResponse response = client().admin().indices().prepareRefresh(index).execute().actionGet(); + assertNoFailures(response); + } finally { + disableIndexBlock(index, blockSetting); + } + } + } + +} diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index e4899c02d37e8..b6dac7bd1596c 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -38,6 +38,8 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.index.codec.customcodecs.ZstdCodec; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; @@ -58,15 +60,21 @@ public class CodecService { public static final String BEST_COMPRESSION_CODEC = "best_compression"; /** the raw unfiltered lucene default. useful for testing */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; + public static final String ZSTD_CODEC = "zstd"; + public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; public CodecService(@Nullable MapperService mapperService, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { codecs.put(DEFAULT_CODEC, new Lucene95Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); + codecs.put(ZSTD_CODEC, new ZstdCodec()); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec()); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java similarity index 65% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java index ad9e5cd3374fa..3c570f9d0566c 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java @@ -8,19 +8,27 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; -import java.util.Locale; - -abstract class Lucene95CustomCodec extends FilterCodec { +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * + * @opensearch.internal + */ +public abstract class Lucene95CustomCodec extends FilterCodec { public static final int DEFAULT_COMPRESSION_LEVEL = 6; /** Each mode represents a compression algorithm. */ public enum Mode { ZSTD, - ZSTDNODICT + ZSTD_NO_DICT } private final StoredFieldsFormat storedFieldsFormat; @@ -40,10 +48,15 @@ public Lucene95CustomCodec(Mode mode) { * the other compression codecs: default, lucene_default, and best_compression. * * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @parama compressionLevel The compression level. + * @param compressionLevel The compression level. */ public Lucene95CustomCodec(Mode mode, int compressionLevel) { - super(mode.name().toLowerCase(Locale.ROOT), new Lucene95Codec()); + super("Lucene95CustomCodec", new Lucene95Codec()); + this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); + } + + public Lucene95CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super("Lucene95CustomCodec", new PerFieldMappingPostingFormatCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, logger)); this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); } diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java similarity index 96% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java index 2bfec2ef171d4..f70306afc8562 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java @@ -8,8 +8,6 @@ package org.opensearch.index.codec.customcodecs; -import java.io.IOException; -import java.util.Objects; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.StoredFieldsWriter; @@ -20,6 +18,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import java.io.IOException; +import java.util.Objects; + /** Stored field format used by pluggable codec */ public class Lucene95CustomStoredFieldsFormat extends StoredFieldsFormat { @@ -95,7 +96,7 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo return impl(mode).fieldsWriter(directory, si, context); } - private StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { + StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { switch (mode) { case ZSTD: return new Lucene90CompressingStoredFieldsFormat( @@ -105,7 +106,7 @@ private StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { ZSTD_MAX_DOCS_PER_BLOCK, ZSTD_BLOCK_SHIFT ); - case ZSTDNODICT: + case ZSTD_NO_DICT: return new Lucene90CompressingStoredFieldsFormat( "CustomStoredFieldsZstdNoDict", zstdNoDictCompressionMode, @@ -117,4 +118,8 @@ private StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { throw new AssertionError(); } } + + Lucene95CustomCodec.Mode getMode() { + return mode; + } } diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java similarity index 79% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java index 2b09540d8037d..68da782421e6e 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java @@ -8,6 +8,9 @@ package org.opensearch.index.codec.customcodecs; +import org.apache.logging.log4j.Logger; +import org.opensearch.index.mapper.MapperService; + /** * ZstdCodec provides ZSTD compressor using the zstd-jni library. */ @@ -29,6 +32,10 @@ public ZstdCodec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } + public ZstdCodec(MapperService mapperService, Logger logger) { + super(Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL, mapperService, logger); + } + /** The name for this codec. */ @Override public String toString() { diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java similarity index 99% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 5b8f1ffcc9569..7057dac3d6bd2 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -13,16 +13,17 @@ import com.github.luben.zstd.ZstdDecompressCtx; import com.github.luben.zstd.ZstdDictCompress; import com.github.luben.zstd.ZstdDictDecompress; -import java.io.IOException; import org.apache.lucene.codecs.compressing.CompressionMode; import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.ByteBuffersDataInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataOutput; -import org.apache.lucene.store.ByteBuffersDataInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import java.io.IOException; + /** Zstandard Compression Mode */ public class ZstdCompressionMode extends CompressionMode { diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java similarity index 74% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java index 4ed6ba57545d0..26620473ec116 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java @@ -8,6 +8,9 @@ package org.opensearch.index.codec.customcodecs; +import org.apache.logging.log4j.Logger; +import org.opensearch.index.mapper.MapperService; + /** * ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ @@ -26,7 +29,11 @@ public ZstdNoDictCodec() { * @param compressionLevel The compression level. */ public ZstdNoDictCodec(int compressionLevel) { - super(Mode.ZSTDNODICT, compressionLevel); + super(Mode.ZSTD_NO_DICT, compressionLevel); + } + + public ZstdNoDictCodec(MapperService mapperService, Logger logger) { + super(Mode.ZSTD_NO_DICT, DEFAULT_COMPRESSION_LEVEL, mapperService, logger); } /** The name for this codec. */ diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java similarity index 99% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index 6cfd85b053190..7a1d661550768 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -9,16 +9,17 @@ package org.opensearch.index.codec.customcodecs; import com.github.luben.zstd.Zstd; -import java.io.IOException; import org.apache.lucene.codecs.compressing.CompressionMode; import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.ByteBuffersDataInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataOutput; -import org.apache.lucene.store.ByteBuffersDataInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import java.io.IOException; + /** ZSTD Compression Mode (without a dictionary support). */ public class ZstdNoDictCompressionMode extends CompressionMode { diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java similarity index 100% rename from sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java rename to server/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 338a541af387a..7419cf1dadea6 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -129,12 +129,14 @@ public Supplier retentionLeasesSupplier() { switch (s) { case "default": case "best_compression": + case "zstd": + case "zstd_no_dict": case "lucene_default": return s; default: if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones throw new IllegalArgumentException( - "unknown value for [index.codec] must be one of [default, best_compression] but was: " + s + "unknown value for [index.codec] must be one of [default, best_compression, zstd, zstd_no_dict] but was: " + s ); } return s; diff --git a/sandbox/plugins/custom-codecs/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec similarity index 100% rename from sandbox/plugins/custom-codecs/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec rename to server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index bc50525412954..016e785f8da13 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -47,6 +47,8 @@ import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.codec.customcodecs.Lucene95CustomCodec; +import org.opensearch.index.codec.customcodecs.Lucene95CustomStoredFieldsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.mapper.MapperRegistry; @@ -63,40 +65,75 @@ public class CodecTests extends OpenSearchTestCase { public void testResolveDefaultCodecs() throws Exception { - CodecService codecService = createCodecService(); + CodecService codecService = createCodecService(false); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class)); } public void testDefault() throws Exception { - Codec codec = createCodecService().codec("default"); + Codec codec = createCodecService(false).codec("default"); assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { - Codec codec = createCodecService().codec("best_compression"); + Codec codec = createCodecService(false).codec("best_compression"); assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); } + public void testZstd() throws Exception { + Codec codec = createCodecService(false).codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + } + + public void testZstdNoDict() throws Exception { + Codec codec = createCodecService(false).codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + } + + public void testDefaultMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("default"); + assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + } + + public void testBestCompressionMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("best_compression"); + assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + } + + public void testZstdMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + } + + public void testZstdNoDictMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + } + + public void testExceptionCodecNull() { + assertThrows(IllegalArgumentException.class, () -> createCodecService(true).codec(null)); + } + // write some docs with it, inspect .si to see this was the used compression private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(actual); - IndexWriter iw = new IndexWriter(dir, iwc); - iw.addDocument(new Document()); - iw.commit(); - iw.close(); - DirectoryReader ir = DirectoryReader.open(dir); - SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); + SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); - ir.close(); - dir.close(); } - private CodecService createCodecService() throws IOException { + private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expected, Codec actual) throws Exception { + SegmentReader sr = getSegmentReader(actual); + String v = sr.getSegmentInfo().info.getAttribute(Lucene95CustomStoredFieldsFormat.MODE_KEY); + assertNotNull(v); + assertEquals(expected, Lucene95CustomCodec.Mode.valueOf(v)); + } + + private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { + + if (isMapperServiceNull) { + return new CodecService(null, LogManager.getLogger("test")); + } Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); @@ -115,4 +152,18 @@ private CodecService createCodecService() throws IOException { return new CodecService(service, LogManager.getLogger("test")); } + private SegmentReader getSegmentReader(Codec codec) throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(null); + iwc.setCodec(codec); + IndexWriter iw = new IndexWriter(dir, iwc); + iw.addDocument(new Document()); + iw.commit(); + iw.close(); + DirectoryReader ir = DirectoryReader.open(dir); + SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); + ir.close(); + dir.close(); + return sr; + } } diff --git a/sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java similarity index 100% rename from sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java rename to server/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java index fcfb06ca6b050..cc794eb2c48f1 100644 --- a/sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java +++ b/server/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java @@ -8,21 +8,21 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.lucene.tests.util.LineFileDocs; -import org.apache.lucene.tests.util.TestUtil; -import org.opensearch.test.OpenSearchTestCase; import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteBuffersDataInput; import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.tests.util.LineFileDocs; +import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; +import org.opensearch.test.OpenSearchTestCase; -import java.util.List; -import java.nio.ByteBuffer; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Random; /** diff --git a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000000000..4f23450ce0b39 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene95CustomCodecMode() { + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(); + assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene95CustomCodecMode() { + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); + } + +} diff --git a/sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java similarity index 100% rename from sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java rename to server/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java diff --git a/sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java similarity index 100% rename from sandbox/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java rename to server/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java From 86f955c2cd45bf8c4fa179c6ff317899dcab925e Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Wed, 28 Jun 2023 17:58:11 -0700 Subject: [PATCH 024/186] Implement buildEmptyAggregations for MultiTermsAggregator (#7089) (#7318) * Implement buildEmptyAggregations for MultiTermsAggregator (#7089) Signed-off-by: Austin Lee * Address Spotless check issue Signed-off-by: Austin Lee * Add a unit test for MultiTermsAggregator.buildEmptyAggregations (#7089) Signed-off-by: Austin Lee * Update changelog Update version check and reason Signed-off-by: Austin Lee --------- Signed-off-by: Austin Lee --- CHANGELOG.md | 1 + .../search.aggregation/370_multi_terms.yml | 48 +++++++++++++ .../bucket/terms/MultiTermsAggregator.java | 15 +++- .../terms/MultiTermsAggregatorTests.java | 69 +++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 139807f0bffd1..faa64a162cbd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Enforce 512 byte document ID limit in bulk updates ([#8039](https://github.com/opensearch-project/OpenSearch/pull/8039)) - With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125)) - Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591)) +- Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml index 0f897866fcb9d..eeab8e78bf830 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml @@ -712,3 +712,51 @@ setup: - match: { aggregations.m_terms.buckets.0.key: ["a", 1] } - match: { aggregations.m_terms.buckets.0.key_as_string: "a|1" } - match: { aggregations.m_terms.buckets.0.doc_count: 4 } + +--- +"aggregate over multi-terms test": + - skip: + version: "- 2.9.99" + reason: "multi_terms aggregation was introduced in 2.1.0, NPE bug checked by this test case will manifest in any version < 3.0" + + - do: + bulk: + index: test_1 + refresh: true + body: + - '{"index": {}}' + - '{"str": "a", "ip": "127.0.0.1", "date": "2022-03-23"}' + - '{"index": {}}' + - '{"str": "a", "ip": "127.0.0.1", "date": "2022-03-25"}' + - '{"index": {}}' + - '{"str": "b", "ip": "127.0.0.1", "date": "2022-03-23"}' + - '{"index": {}}' + - '{"str": "b", "ip": "127.0.0.1", "date": "2022-03-25"}' + + - do: + search: + index: test_1 + size: 0 + body: + aggs: + histo: + date_histogram: + field: date + calendar_interval: day + aggs: + m_terms: + multi_terms: + terms: + - field: str + - field: ip + + - match: { hits.total.value: 4 } + - length: { aggregations.histo.buckets: 3 } + - match: { aggregations.histo.buckets.0.key_as_string: "2022-03-23T00:00:00.000Z" } + - match: { aggregations.histo.buckets.0.m_terms.buckets.0.key: ["a", "127.0.0.1"] } + - match: { aggregations.histo.buckets.0.m_terms.buckets.1.key: ["b", "127.0.0.1"] } + - match: { aggregations.histo.buckets.1.key_as_string: "2022-03-24T00:00:00.000Z" } + - length: { aggregations.histo.buckets.1.m_terms.buckets: 0 } + - match: { aggregations.histo.buckets.2.key_as_string: "2022-03-25T00:00:00.000Z" } + - match: { aggregations.histo.buckets.2.m_terms.buckets.0.key: [ "a", "127.0.0.1" ] } + - match: { aggregations.histo.buckets.2.m_terms.buckets.1.key: [ "b", "127.0.0.1" ] } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java index c810ba8f38624..fccb9c3af5986 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java @@ -196,7 +196,20 @@ InternalMultiTerms buildResult(long owningBucketOrd, long otherDocCount, Interna @Override public InternalAggregation buildEmptyAggregation() { - return null; + return new InternalMultiTerms( + name, + order, + order, + bucketCountThresholds.getRequiredSize(), + bucketCountThresholds.getMinDocCount(), + metadata(), + bucketCountThresholds.getShardSize(), + showTermDocCountError, + 0, + 0, + formats, + Collections.emptyList() + ); } @Override diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java index 75ad9e12e0776..a2792114e9529 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java @@ -28,6 +28,10 @@ import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.MockPageCacheRecycler; +import org.opensearch.index.IndexService; +import org.opensearch.index.cache.IndexCache; import org.opensearch.index.mapper.BooleanFieldMapper; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.GeoPointFieldMapper; @@ -35,22 +39,32 @@ import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptModule; import org.opensearch.script.ScriptService; import org.opensearch.script.ScriptType; +import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.CardinalityUpperBound; +import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.aggregations.support.ValuesSourceType; +import org.opensearch.search.internal.SearchContext; import org.opensearch.search.lookup.LeafDocLookup; +import org.opensearch.test.TestSearchContext; import java.io.IOException; import java.util.ArrayList; @@ -58,6 +72,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.function.Consumer; import java.util.function.Function; @@ -68,8 +83,12 @@ import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MultiTermsAggregatorTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; @@ -852,6 +871,56 @@ public void testIncludeExclude() throws IOException { ); } + public void testEmptyAggregations() throws IOException { + QueryShardContext queryShardContext = mock(QueryShardContext.class); + IndexShard indexShard = mock(IndexShard.class); + BigArrays bigArrays = new BigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService(), ""); + IndexService indexService = mock(IndexService.class); + when(indexService.getShardOrNull(0)).thenReturn(indexShard); + IndexCache cache = mock(IndexCache.class); + when(cache.bitsetFilterCache()).thenReturn(null); + when(indexService.cache()).thenReturn(cache); + SearchContext context = new TestSearchContext(bigArrays, indexService); + when(indexService.newQueryShardContext(0, null, () -> 0L, null)).thenReturn(queryShardContext); + AggregatorFactories factories = AggregatorFactories.EMPTY; + boolean showTermDocCountError = true; + MultiTermsAggregator.InternalValuesSource internalValuesSources = mock(MultiTermsAggregator.InternalValuesSource.class); + DocValueFormat format = mock(DocValueFormat.class); + BucketOrder order = mock(BucketOrder.class); + Aggregator.SubAggCollectionMode collectMode = Aggregator.SubAggCollectionMode.BREADTH_FIRST; + TermsAggregator.BucketCountThresholds bucketCountThresholds = mock(TermsAggregator.BucketCountThresholds.class); + Aggregator parent = mock(Aggregator.class); + CardinalityUpperBound cardinality = CardinalityUpperBound.ONE; + Map metadata = new HashMap<>(); + String k1 = UUID.randomUUID().toString(); + String v1 = UUID.randomUUID().toString(); + metadata.put(k1, v1); + + MultiTermsAggregator mAgg = new MultiTermsAggregator( + AGG_NAME, + factories, + showTermDocCountError, + List.of(internalValuesSources), + List.of(format), + order, + collectMode, + bucketCountThresholds, + context, + parent, + cardinality, + metadata + ); + InternalAggregation emptyAgg = mAgg.buildEmptyAggregation(); + + MatcherAssert.assertThat(emptyAgg.getName(), equalTo(AGG_NAME)); + MatcherAssert.assertThat(emptyAgg, instanceOf(InternalMultiTerms.class)); + + InternalMultiTerms mt = (InternalMultiTerms) emptyAgg; + MatcherAssert.assertThat(mt.getMetadata().keySet(), contains(k1)); + MatcherAssert.assertThat(mt.getMetadata().get(k1), equalTo(v1)); + MatcherAssert.assertThat(mt.getBuckets(), empty()); + } + private void testAggregation( Query query, List terms, From b33979a4d95f728ee3080eb6983d93e024fb7ba2 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 28 Jun 2023 18:01:17 -0700 Subject: [PATCH 025/186] Adding the SearchPhaseResultsProcessor interface in Search Pipeline (#7283) * Initial code for adding the SearchPhaseInjectorProcessor interface in Search Pipeline Signed-off-by: Navneet Verma * Pass PipelinedRequest to SearchAsyncActions We should resolve a search pipeline once at the start of a search request and then propagate that pipeline through the async actions. When completing a search phase, we will then use that pipeline to inject behavior (if applicable). Signed-off-by: Michael Froh * Renamed SearchPhaseInjectorProcessor to SearchPhaseResultsProcessor and fixed the comments Signed-off-by: Navneet Verma * Make PipelinedSearchRequest extend SearchRequest Rather than wrapping a SearchRequest in a PipelinedSearchRequest, changes are less intrusive if we say that a PipelinedSearchRequest "is a" SearchRequest. Signed-off-by: Michael Froh * Revert code change from merge conflict Signed-off-by: Michael Froh * Updated the changelog with more appropiate wording for the change. Signed-off-by: Navneet Verma * Fixed Typos in the code Signed-off-by: Navneet Verma * Fixing comments relating to return of SearchPhaseResults from processor Signed-off-by: Navneet Verma * Moved SearchPhaseName enum in separate class and fixed comments. Signed-off-by: Navneet Verma * Resolve remaining merge conflict Signed-off-by: Michael Froh --------- Signed-off-by: Navneet Verma Signed-off-by: Michael Froh Co-authored-by: Michael Froh Co-authored-by: Andrew Ross --- CHANGELOG.md | 1 + .../search_pipeline/50_script_processor.yml | 2 +- .../search/AbstractSearchAsyncAction.java | 7 +- .../search/ArraySearchPhaseResults.java | 2 +- .../search/CanMatchPreFilterSearchPhase.java | 2 +- .../action/search/DfsQueryPhase.java | 2 +- .../action/search/ExpandSearchPhase.java | 2 +- .../action/search/FetchSearchPhase.java | 2 +- .../opensearch/action/search/SearchPhase.java | 10 + .../action/search/SearchPhaseContext.java | 2 +- .../action/search/SearchPhaseName.java | 31 +++ .../action/search/SearchPhaseResults.java | 10 +- .../search/SearchScrollAsyncAction.java | 2 +- ...SearchScrollQueryThenFetchAsyncAction.java | 2 +- .../action/search/TransportSearchAction.java | 7 +- .../plugins/SearchPipelinePlugin.java | 12 + .../opensearch/search/pipeline/Pipeline.java | 33 ++- .../search/pipeline/PipelineWithMetrics.java | 24 +- .../search/pipeline/PipelinedRequest.java | 19 +- .../pipeline/SearchPhaseResultsProcessor.java | 47 ++++ .../pipeline/SearchPipelineService.java | 10 +- .../pipeline/SearchPipelineServiceTests.java | 219 ++++++++++++++++-- 22 files changed, 402 insertions(+), 46 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/search/SearchPhaseName.java create mode 100644 server/src/main/java/org/opensearch/search/pipeline/SearchPhaseResultsProcessor.java diff --git a/CHANGELOG.md b/CHANGELOG.md index faa64a162cbd8..93b2296475fd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- [SearchPipeline] Add new search pipeline processor type, SearchPhaseResultsProcessor, that can modify the result of one search phase before starting the next phase.([#7283](https://github.com/opensearch-project/OpenSearch/pull/7283)) - Add task cancellation monitoring service ([#7642](https://github.com/opensearch-project/OpenSearch/pull/7642)) - Add TokenManager Interface ([#7452](https://github.com/opensearch-project/OpenSearch/pull/7452)) - Add Remote store as a segment replication source ([#7653](https://github.com/opensearch-project/OpenSearch/pull/7653)) diff --git a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml index 9b2dc0c41ff31..9d855e8a1861a 100644 --- a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml +++ b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml @@ -39,7 +39,7 @@ teardown: { "script" : { "lang" : "painless", - "source" : "ctx._source['size'] += 10; ctx._source['from'] -= 1; ctx._source['explain'] = !ctx._source['explain']; ctx._source['version'] = !ctx._source['version']; ctx._source['seq_no_primary_term'] = !ctx._source['seq_no_primary_term']; ctx._source['track_scores'] = !ctx._source['track_scores']; ctx._source['track_total_hits'] = 1; ctx._source['min_score'] -= 0.9; ctx._source['terminate_after'] += 2; ctx._source['profile'] = !ctx._source['profile'];" + "source" : "ctx._source['size'] += 10; ctx._source['from'] = ctx._source['from'] <= 0 ? ctx._source['from'] : ctx._source['from'] - 1 ; ctx._source['explain'] = !ctx._source['explain']; ctx._source['version'] = !ctx._source['version']; ctx._source['seq_no_primary_term'] = !ctx._source['seq_no_primary_term']; ctx._source['track_scores'] = !ctx._source['track_scores']; ctx._source['track_total_hits'] = 1; ctx._source['min_score'] -= 0.9; ctx._source['terminate_after'] += 2; ctx._source['profile'] = !ctx._source['profile'];" } } ] diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index 969e0edbbc9d6..48fac9e8c8d38 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -57,6 +57,7 @@ import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.search.internal.SearchContext; import org.opensearch.search.internal.ShardSearchRequest; +import org.opensearch.search.pipeline.PipelinedRequest; import org.opensearch.transport.Transport; import java.util.ArrayDeque; @@ -696,7 +697,11 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { * @see #onShardResult(SearchPhaseResult, SearchShardIterator) */ final void onPhaseDone() { // as a tribute to @kimchy aka. finishHim() - executeNextPhase(this, getNextPhase(results, this)); + final SearchPhase nextPhase = getNextPhase(results, this); + if (request instanceof PipelinedRequest && nextPhase != null) { + ((PipelinedRequest) request).transformSearchPhaseResults(results, this, this.getName(), nextPhase.getName()); + } + executeNextPhase(this, nextPhase); } @Override diff --git a/server/src/main/java/org/opensearch/action/search/ArraySearchPhaseResults.java b/server/src/main/java/org/opensearch/action/search/ArraySearchPhaseResults.java index 61c81e6cda97a..653b0e8aedb9d 100644 --- a/server/src/main/java/org/opensearch/action/search/ArraySearchPhaseResults.java +++ b/server/src/main/java/org/opensearch/action/search/ArraySearchPhaseResults.java @@ -66,7 +66,7 @@ boolean hasResult(int shardIndex) { } @Override - AtomicArray getAtomicArray() { + public AtomicArray getAtomicArray() { return results; } } diff --git a/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java index ec4d45a0a7124..c026c72f77f00 100644 --- a/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java @@ -94,7 +94,7 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction, SearchPhase> nextPhaseFactory, SearchPhaseContext context ) { - super("dfs_query"); + super(SearchPhaseName.DFS_QUERY.getName()); this.progressListener = context.getTask().getProgressListener(); this.queryResult = queryResult; this.searchResults = searchResults; diff --git a/server/src/main/java/org/opensearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/opensearch/action/search/ExpandSearchPhase.java index cdefe7c2c1712..618a5620ce093 100644 --- a/server/src/main/java/org/opensearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/ExpandSearchPhase.java @@ -62,7 +62,7 @@ final class ExpandSearchPhase extends SearchPhase { private final AtomicArray queryResults; ExpandSearchPhase(SearchPhaseContext context, InternalSearchResponse searchResponse, AtomicArray queryResults) { - super("expand"); + super(SearchPhaseName.EXPAND.getName()); this.context = context; this.searchResponse = searchResponse; this.queryResults = queryResults; diff --git a/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java index 31ec896856ce6..85a3d140977bb 100644 --- a/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java @@ -92,7 +92,7 @@ final class FetchSearchPhase extends SearchPhase { SearchPhaseContext context, BiFunction, SearchPhase> nextPhaseFactory ) { - super("fetch"); + super(SearchPhaseName.FETCH.getName()); if (context.getNumShards() != resultConsumer.getNumShards()) { throw new IllegalStateException( "number of shards must match the length of the query results but doesn't:" diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhase.java b/server/src/main/java/org/opensearch/action/search/SearchPhase.java index 50f0940754078..50b0cd8e01c1d 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhase.java @@ -34,6 +34,7 @@ import org.opensearch.common.CheckedRunnable; import java.io.IOException; +import java.util.Locale; import java.util.Objects; /** @@ -54,4 +55,13 @@ protected SearchPhase(String name) { public String getName() { return name; } + + /** + * Returns the SearchPhase name as {@link SearchPhaseName}. Exception will come if SearchPhase name is not defined + * in {@link SearchPhaseName} + * @return {@link SearchPhaseName} + */ + public SearchPhaseName getSearchPhaseName() { + return SearchPhaseName.valueOf(name.toUpperCase(Locale.ROOT)); + } } diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java index 04b481249520b..018035f21179b 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java @@ -50,7 +50,7 @@ * * @opensearch.internal */ -interface SearchPhaseContext extends Executor { +public interface SearchPhaseContext extends Executor { // TODO maybe we can make this concrete later - for now we just implement this in the base class for all initial phases /** diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java new file mode 100644 index 0000000000000..b6f842cf2cce1 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +/** + * Enum for different Search Phases in OpenSearch + * @opensearch.internal + */ +public enum SearchPhaseName { + QUERY("query"), + FETCH("fetch"), + DFS_QUERY("dfs_query"), + EXPAND("expand"), + CAN_MATCH("can_match"); + + private final String name; + + SearchPhaseName(final String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseResults.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseResults.java index 1baea0e721c44..2e6068b1ecddc 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseResults.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseResults.java @@ -42,7 +42,7 @@ * * @opensearch.internal */ -abstract class SearchPhaseResults { +public abstract class SearchPhaseResults { private final int numShards; SearchPhaseResults(int numShards) { @@ -75,7 +75,13 @@ final int getNumShards() { void consumeShardFailure(int shardIndex) {} - AtomicArray getAtomicArray() { + /** + * Returns an {@link AtomicArray} of {@link Result}, which are nothing but the SearchPhaseResults + * for shards. The {@link Result} are of type {@link SearchPhaseResult} + * + * @return an {@link AtomicArray} of {@link Result} + */ + public AtomicArray getAtomicArray() { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/opensearch/action/search/SearchScrollAsyncAction.java index 0b477624b15cc..899c7a3c1dabd 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchScrollAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/SearchScrollAsyncAction.java @@ -266,7 +266,7 @@ protected SearchPhase sendResponsePhase( SearchPhaseController.ReducedQueryPhase queryPhase, final AtomicArray fetchResults ) { - return new SearchPhase("fetch") { + return new SearchPhase(SearchPhaseName.FETCH.getName()) { @Override public void run() throws IOException { sendResponse(queryPhase, fetchResults); diff --git a/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java index 4119cb1cf28a0..9c0721ef63ea6 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java @@ -92,7 +92,7 @@ protected void executeInitialPhase( @Override protected SearchPhase moveToNextPhase(BiFunction clusterNodeLookup) { - return new SearchPhase("fetch") { + return new SearchPhase(SearchPhaseName.FETCH.getName()) { @Override public void run() { final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = searchPhaseController.reducedScrollQueryPhase( diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 69f529fe1d00c..df2170cbe2af1 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -390,13 +390,12 @@ private void executeRequest( relativeStartNanos, System::nanoTime ); - SearchRequest searchRequest; + PipelinedRequest searchRequest; ActionListener listener; try { - PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(originalSearchRequest); - searchRequest = pipelinedRequest.transformedRequest(); + searchRequest = searchPipelineService.resolvePipeline(originalSearchRequest); listener = ActionListener.wrap( - r -> originalListener.onResponse(pipelinedRequest.transformResponse(r)), + r -> originalListener.onResponse(searchRequest.transformResponse(r)), originalListener::onFailure ); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java index b8ceddecd3d20..3d76bab93a60c 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java @@ -9,6 +9,7 @@ package org.opensearch.plugins; import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.SearchPhaseResultsProcessor; import org.opensearch.search.pipeline.SearchRequestProcessor; import org.opensearch.search.pipeline.SearchResponseProcessor; @@ -42,4 +43,15 @@ default Map> getRequestProcess default Map> getResponseProcessors(Processor.Parameters parameters) { return Collections.emptyMap(); } + + /** + * Returns additional search pipeline search phase results processor types added by this plugin. + * + * The key of the returned {@link Map} is the unique name for the processor which is specified + * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory} + * to create the processor from a given pipeline configuration. + */ + default Map> getSearchPhaseResultsProcessors(Processor.Parameters parameters) { + return Collections.emptyMap(); + } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java index 6f44daf48ed21..92826eee5a4f4 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java +++ b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java @@ -8,6 +8,8 @@ package org.opensearch.search.pipeline; +import org.opensearch.action.search.SearchPhaseContext; +import org.opensearch.action.search.SearchPhaseResults; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Nullable; @@ -15,6 +17,7 @@ import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.search.SearchPhaseResult; import java.util.Collections; import java.util.List; @@ -28,6 +31,7 @@ class Pipeline { public static final String REQUEST_PROCESSORS_KEY = "request_processors"; public static final String RESPONSE_PROCESSORS_KEY = "response_processors"; + public static final String PHASE_PROCESSORS_KEY = "phase_results_processors"; private final String id; private final String description; private final Integer version; @@ -36,7 +40,7 @@ class Pipeline { // Then these can be CompoundProcessors instead of lists. private final List searchRequestProcessors; private final List searchResponseProcessors; - + private final List searchPhaseResultsProcessors; private final NamedWriteableRegistry namedWriteableRegistry; private final LongSupplier relativeTimeSupplier; @@ -46,6 +50,7 @@ class Pipeline { @Nullable Integer version, List requestProcessors, List responseProcessors, + List phaseResultsProcessors, NamedWriteableRegistry namedWriteableRegistry, LongSupplier relativeTimeSupplier ) { @@ -54,6 +59,7 @@ class Pipeline { this.version = version; this.searchRequestProcessors = Collections.unmodifiableList(requestProcessors); this.searchResponseProcessors = Collections.unmodifiableList(responseProcessors); + this.searchPhaseResultsProcessors = Collections.unmodifiableList(phaseResultsProcessors); this.namedWriteableRegistry = namedWriteableRegistry; this.relativeTimeSupplier = relativeTimeSupplier; } @@ -78,6 +84,10 @@ List getSearchResponseProcessors() { return searchResponseProcessors; } + List getSearchPhaseResultsProcessors() { + return searchPhaseResultsProcessors; + } + protected void beforeTransformRequest() {} protected void afterTransformRequest(long timeInNanos) {} @@ -168,14 +178,33 @@ SearchResponse transformResponse(SearchRequest request, SearchResponse response) return response; } + void runSearchPhaseResultsTransformer( + SearchPhaseResults searchPhaseResult, + SearchPhaseContext context, + String currentPhase, + String nextPhase + ) throws SearchPipelineProcessingException { + + try { + for (SearchPhaseResultsProcessor searchPhaseResultsProcessor : searchPhaseResultsProcessors) { + if (currentPhase.equals(searchPhaseResultsProcessor.getBeforePhase().getName()) + && nextPhase.equals(searchPhaseResultsProcessor.getAfterPhase().getName())) { + searchPhaseResultsProcessor.process(searchPhaseResult, context); + } + } + } catch (RuntimeException e) { + throw new SearchPipelineProcessingException(e); + } + } + static final Pipeline NO_OP_PIPELINE = new Pipeline( SearchPipelineService.NOOP_PIPELINE_ID, "Pipeline that does not transform anything", 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), null, () -> 0L ); - } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java index 662473f190006..612e979e56070 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java @@ -43,12 +43,22 @@ class PipelineWithMetrics extends Pipeline { Integer version, List requestProcessors, List responseProcessors, + List phaseResultsProcessors, NamedWriteableRegistry namedWriteableRegistry, OperationMetrics totalRequestMetrics, OperationMetrics totalResponseMetrics, LongSupplier relativeTimeSupplier ) { - super(id, description, version, requestProcessors, responseProcessors, namedWriteableRegistry, relativeTimeSupplier); + super( + id, + description, + version, + requestProcessors, + responseProcessors, + phaseResultsProcessors, + namedWriteableRegistry, + relativeTimeSupplier + ); this.totalRequestMetrics = totalRequestMetrics; this.totalResponseMetrics = totalResponseMetrics; for (Processor requestProcessor : getSearchRequestProcessors()) { @@ -64,6 +74,7 @@ static PipelineWithMetrics create( Map config, Map> requestProcessorFactories, Map> responseProcessorFactories, + Map> phaseResultsProcessorFactories, NamedWriteableRegistry namedWriteableRegistry, OperationMetrics totalRequestProcessingMetrics, OperationMetrics totalResponseProcessingMetrics @@ -79,6 +90,16 @@ static PipelineWithMetrics create( RESPONSE_PROCESSORS_KEY ); List responseProcessors = readProcessors(responseProcessorFactories, responseProcessorConfigs); + List> phaseResultsProcessorConfigs = ConfigurationUtils.readOptionalList( + null, + null, + config, + PHASE_PROCESSORS_KEY + ); + List phaseResultsProcessors = readProcessors( + phaseResultsProcessorFactories, + phaseResultsProcessorConfigs + ); if (config.isEmpty() == false) { throw new OpenSearchParseException( "pipeline [" @@ -93,6 +114,7 @@ static PipelineWithMetrics create( version, requestProcessors, responseProcessors, + phaseResultsProcessors, namedWriteableRegistry, totalRequestProcessingMetrics, totalResponseProcessingMetrics, diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java b/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java index 0cfff013f4021..5a7539808c127 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java @@ -8,29 +8,36 @@ package org.opensearch.search.pipeline; +import org.opensearch.action.search.SearchPhaseContext; +import org.opensearch.action.search.SearchPhaseResults; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchPhaseResult; /** * Groups a search pipeline based on a request and the request after being transformed by the pipeline. * * @opensearch.internal */ -public final class PipelinedRequest { +public final class PipelinedRequest extends SearchRequest { private final Pipeline pipeline; - private final SearchRequest transformedRequest; PipelinedRequest(Pipeline pipeline, SearchRequest transformedRequest) { + super(transformedRequest); this.pipeline = pipeline; - this.transformedRequest = transformedRequest; } public SearchResponse transformResponse(SearchResponse response) { - return pipeline.transformResponse(transformedRequest, response); + return pipeline.transformResponse(this, response); } - public SearchRequest transformedRequest() { - return transformedRequest; + public void transformSearchPhaseResults( + final SearchPhaseResults searchPhaseResult, + final SearchPhaseContext searchPhaseContext, + final String currentPhase, + final String nextPhase + ) { + pipeline.runSearchPhaseResultsTransformer(searchPhaseResult, searchPhaseContext, currentPhase, nextPhase); } // Visible for testing diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPhaseResultsProcessor.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPhaseResultsProcessor.java new file mode 100644 index 0000000000000..772dc8758bace --- /dev/null +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPhaseResultsProcessor.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline; + +import org.opensearch.action.search.SearchPhaseContext; +import org.opensearch.action.search.SearchPhaseName; +import org.opensearch.action.search.SearchPhaseResults; +import org.opensearch.search.SearchPhaseResult; +import org.opensearch.search.internal.SearchContext; + +/** + * Creates a processor that runs between Phases of the Search. + * @opensearch.api + */ +public interface SearchPhaseResultsProcessor extends Processor { + + /** + * Processes the {@link SearchPhaseResults} obtained from a SearchPhase which will be returned to next + * SearchPhase. + * @param searchPhaseResult {@link SearchPhaseResults} + * @param searchPhaseContext {@link SearchContext} + * @param {@link SearchPhaseResult} + */ + void process( + final SearchPhaseResults searchPhaseResult, + final SearchPhaseContext searchPhaseContext + ); + + /** + * The phase which should have run before, this processor can start executing. + * @return {@link SearchPhaseName} + */ + SearchPhaseName getBeforePhase(); + + /** + * The phase which should run after, this processor execution. + * @return {@link SearchPhaseName} + */ + SearchPhaseName getAfterPhase(); + +} diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 434c8fbfacc74..70dc8546a077f 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -73,6 +73,7 @@ public class SearchPipelineService implements ClusterStateApplier, ReportingServ private final ScriptService scriptService; private final Map> requestProcessorFactories; private final Map> responseProcessorFactories; + private final Map> phaseInjectorProcessorFactories; private volatile Map pipelines = Collections.emptyMap(); private final ThreadPool threadPool; private final List> searchPipelineClusterStateListeners = new CopyOnWriteArrayList<>(); @@ -116,6 +117,10 @@ public SearchPipelineService( ); this.requestProcessorFactories = processorFactories(searchPipelinePlugins, p -> p.getRequestProcessors(parameters)); this.responseProcessorFactories = processorFactories(searchPipelinePlugins, p -> p.getResponseProcessors(parameters)); + this.phaseInjectorProcessorFactories = processorFactories( + searchPipelinePlugins, + p -> p.getSearchPhaseResultsProcessors(parameters) + ); putPipelineTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.PUT_SEARCH_PIPELINE_KEY, true); deletePipelineTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.DELETE_SEARCH_PIPELINE_KEY, true); this.isEnabled = isEnabled; @@ -181,6 +186,7 @@ void innerUpdatePipelines(SearchPipelineMetadata newSearchPipelineMetadata) { newConfiguration.getConfigAsMap(), requestProcessorFactories, responseProcessorFactories, + phaseInjectorProcessorFactories, namedWriteableRegistry, totalRequestProcessingMetrics, totalResponseProcessingMetrics @@ -280,6 +286,7 @@ void validatePipeline(Map searchPipelineInfos pipelineConfig, requestProcessorFactories, responseProcessorFactories, + phaseInjectorProcessorFactories, namedWriteableRegistry, new OperationMetrics(), // Use ephemeral metrics for validation new OperationMetrics() @@ -359,7 +366,7 @@ static ClusterState innerDelete(DeleteSearchPipelineRequest request, ClusterStat return newState.build(); } - public PipelinedRequest resolvePipeline(SearchRequest searchRequest) throws Exception { + public PipelinedRequest resolvePipeline(SearchRequest searchRequest) { Pipeline pipeline = Pipeline.NO_OP_PIPELINE; if (isEnabled == false) { @@ -378,6 +385,7 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) throws Exce searchRequest.source().searchPipelineSource(), requestProcessorFactories, responseProcessorFactories, + phaseInjectorProcessorFactories, namedWriteableRegistry, totalRequestProcessingMetrics, totalResponseProcessingMetrics diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index 219dddff40b35..2ac0b2136ddd9 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -10,13 +10,22 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.junit.Before; import org.opensearch.OpenSearchParseException; import org.opensearch.ResourceNotFoundException; import org.opensearch.Version; import org.opensearch.action.search.DeleteSearchPipelineRequest; +import org.opensearch.action.search.MockSearchPhaseContext; import org.opensearch.action.search.PutSearchPipelineRequest; +import org.opensearch.action.search.QueryPhaseResultConsumer; +import org.opensearch.action.search.SearchPhaseContext; +import org.opensearch.action.search.SearchPhaseController; +import org.opensearch.action.search.SearchPhaseName; +import org.opensearch.action.search.SearchPhaseResults; +import org.opensearch.action.search.SearchProgressListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchResponseSections; @@ -28,10 +37,14 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.common.breaker.NoopCircuitBreaker; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexSettings; @@ -40,7 +53,10 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchModule; +import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.test.InternalAggregationTestCase; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; @@ -68,6 +84,13 @@ public Map> getRequestProcesso public Map> getResponseProcessors(Processor.Parameters parameters) { return Map.of("bar", (factories, tag, description, config) -> null); } + + @Override + public Map> getSearchPhaseResultsProcessors( + Processor.Parameters parameters + ) { + return Map.of("zoe", (factories, tag, description, config) -> null); + } }; private ThreadPool threadPool; @@ -178,13 +201,13 @@ public void testResolveIndexDefaultPipeline() throws Exception { SearchRequest searchRequest = new SearchRequest("my_index").source(SearchSourceBuilder.searchSource().size(5)); PipelinedRequest pipelinedRequest = service.resolvePipeline(searchRequest); assertEquals("p1", pipelinedRequest.getPipeline().getId()); - assertEquals(10, pipelinedRequest.transformedRequest().source().size()); + assertEquals(10, pipelinedRequest.source().size()); // Bypass the default pipeline searchRequest.pipeline("_none"); pipelinedRequest = service.resolvePipeline(searchRequest); assertEquals("_none", pipelinedRequest.getPipeline().getId()); - assertEquals(5, pipelinedRequest.transformedRequest().source().size()); + assertEquals(5, pipelinedRequest.source().size()); } private static abstract class FakeProcessor implements Processor { @@ -244,6 +267,40 @@ public SearchResponse processResponse(SearchRequest request, SearchResponse resp } } + private static class FakeSearchPhaseResultsProcessor extends FakeProcessor implements SearchPhaseResultsProcessor { + private Consumer querySearchResultConsumer; + + public FakeSearchPhaseResultsProcessor( + String type, + String tag, + String description, + Consumer querySearchResultConsumer + ) { + super(type, tag, description); + this.querySearchResultConsumer = querySearchResultConsumer; + } + + @Override + public void process( + SearchPhaseResults searchPhaseResult, + SearchPhaseContext searchPhaseContext + ) { + List resultAtomicArray = searchPhaseResult.getAtomicArray().asList(); + // updating the maxScore + resultAtomicArray.forEach(querySearchResultConsumer); + } + + @Override + public SearchPhaseName getBeforePhase() { + return SearchPhaseName.QUERY; + } + + @Override + public SearchPhaseName getAfterPhase() { + return SearchPhaseName.FETCH; + } + } + private SearchPipelineService createWithProcessors() { Map> requestProcessors = new HashMap<>(); requestProcessors.put("scale_request_size", (processorFactories, tag, description, config) -> { @@ -260,7 +317,15 @@ private SearchPipelineService createWithProcessors() { float score = ((Number) config.remove("score")).floatValue(); return new FakeResponseProcessor("fixed_score", tag, description, rsp -> rsp.getHits().forEach(h -> h.score(score))); }); - return createWithProcessors(requestProcessors, responseProcessors); + + Map> searchPhaseProcessors = new HashMap<>(); + searchPhaseProcessors.put("max_score", (processorFactories, tag, description, config) -> { + final float finalScore = config.containsKey("score") ? ((Number) config.remove("score")).floatValue() : 100f; + final Consumer querySearchResultConsumer = (result) -> result.queryResult().topDocs().maxScore = finalScore; + return new FakeSearchPhaseResultsProcessor("max_score", tag, description, querySearchResultConsumer); + }); + + return createWithProcessors(requestProcessors, responseProcessors, searchPhaseProcessors); } @Override @@ -271,7 +336,8 @@ protected NamedWriteableRegistry writableRegistry() { private SearchPipelineService createWithProcessors( Map> requestProcessors, - Map> responseProcessors + Map> responseProcessors, + Map> phaseProcessors ) { Client client = mock(Client.class); ThreadPool threadPool = mock(ThreadPool.class); @@ -296,6 +362,14 @@ public Map> getRequestProcesso public Map> getResponseProcessors(Processor.Parameters parameters) { return responseProcessors; } + + @Override + public Map> getSearchPhaseResultsProcessors( + Processor.Parameters parameters + ) { + return phaseProcessors; + } + }), client, true @@ -314,7 +388,8 @@ public void testUpdatePipelines() { new BytesArray( "{ " + "\"request_processors\" : [ { \"scale_request_size\": { \"scale\" : 2 } } ], " - + "\"response_processors\" : [ { \"fixed_score\" : { \"score\" : 1.0 } } ]" + + "\"response_processors\" : [ { \"fixed_score\" : { \"score\" : 1.0 } } ]," + + "\"phase_results_processors\" : [ { \"max_score\" : { \"score\": 100 } } ]" + "}" ), XContentType.JSON @@ -332,6 +407,11 @@ public void testUpdatePipelines() { "scale_request_size", searchPipelineService.getPipelines().get("_id").pipeline.getSearchRequestProcessors().get(0).getType() ); + assertEquals(1, searchPipelineService.getPipelines().get("_id").pipeline.getSearchPhaseResultsProcessors().size()); + assertEquals( + "max_score", + searchPipelineService.getPipelines().get("_id").pipeline.getSearchPhaseResultsProcessors().get(0).getType() + ); assertEquals(1, searchPipelineService.getPipelines().get("_id").pipeline.getSearchResponseProcessors().size()); assertEquals( "fixed_score", @@ -369,6 +449,7 @@ public void testPutPipeline() { assertEquals("empty pipeline", pipeline.pipeline.getDescription()); assertEquals(0, pipeline.pipeline.getSearchRequestProcessors().size()); assertEquals(0, pipeline.pipeline.getSearchResponseProcessors().size()); + assertEquals(0, pipeline.pipeline.getSearchPhaseResultsProcessors().size()); } public void testPutInvalidPipeline() throws IllegalAccessException { @@ -506,17 +587,14 @@ public void testTransformRequest() throws Exception { SearchRequest request = new SearchRequest("_index").source(sourceBuilder).pipeline("p1"); PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(request); - SearchRequest transformedRequest = pipelinedRequest.transformedRequest(); - assertEquals(2 * size, transformedRequest.source().size()); + assertEquals(2 * size, pipelinedRequest.source().size()); assertEquals(size, request.source().size()); // This request doesn't specify a pipeline, it doesn't get transformed. request = new SearchRequest("_index").source(sourceBuilder); pipelinedRequest = searchPipelineService.resolvePipeline(request); - SearchRequest notTransformedRequest = pipelinedRequest.transformedRequest(); - assertEquals(size, notTransformedRequest.source().size()); - assertSame(request, notTransformedRequest); + assertEquals(size, pipelinedRequest.source().size()); } public void testTransformResponse() throws Exception { @@ -565,6 +643,89 @@ public void testTransformResponse() throws Exception { } } + public void testTransformSearchPhase() { + SearchPipelineService searchPipelineService = createWithProcessors(); + SearchPipelineMetadata metadata = new SearchPipelineMetadata( + Map.of( + "p1", + new PipelineConfiguration( + "p1", + new BytesArray("{\"phase_results_processors\" : [ { \"max_score\" : { } } ]}"), + XContentType.JSON + ) + ) + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousState = clusterState; + clusterState = ClusterState.builder(clusterState) + .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata)) + .build(); + searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState)); + SearchPhaseController controller = new SearchPhaseController( + writableRegistry(), + s -> InternalAggregationTestCase.emptyReduceContextBuilder() + ); + SearchPhaseContext searchPhaseContext = new MockSearchPhaseContext(10); + QueryPhaseResultConsumer searchPhaseResults = new QueryPhaseResultConsumer( + searchPhaseContext.getRequest(), + OpenSearchExecutors.newDirectExecutorService(), + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + SearchProgressListener.NOOP, + writableRegistry(), + 2, + exc -> {} + ); + + final QuerySearchResult querySearchResult = new QuerySearchResult(); + querySearchResult.setShardIndex(1); + querySearchResult.topDocs(new TopDocsAndMaxScore(new TopDocs(null, new ScoreDoc[1]), 1f), null); + searchPhaseResults.consumeResult(querySearchResult, () -> {}); + + // First try without specifying a pipeline, which should be a no-op. + SearchRequest searchRequest = new SearchRequest(); + PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest); + AtomicArray notTransformedSearchPhaseResults = searchPhaseResults.getAtomicArray(); + pipelinedRequest.transformSearchPhaseResults( + searchPhaseResults, + searchPhaseContext, + SearchPhaseName.QUERY.getName(), + SearchPhaseName.FETCH.getName() + ); + assertSame(searchPhaseResults.getAtomicArray(), notTransformedSearchPhaseResults); + + // Now set the pipeline as p1 + searchRequest = new SearchRequest().pipeline("p1"); + pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest); + + pipelinedRequest.transformSearchPhaseResults( + searchPhaseResults, + searchPhaseContext, + SearchPhaseName.QUERY.getName(), + SearchPhaseName.FETCH.getName() + ); + + List resultAtomicArray = searchPhaseResults.getAtomicArray().asList(); + assertEquals(1, resultAtomicArray.size()); + // updating the maxScore + for (SearchPhaseResult result : resultAtomicArray) { + assertEquals(100f, result.queryResult().topDocs().maxScore, 0); + } + + // Check Processor doesn't run for between other phases + searchRequest = new SearchRequest().pipeline("p1"); + pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest); + AtomicArray notTransformedSearchPhaseResult = searchPhaseResults.getAtomicArray(); + pipelinedRequest.transformSearchPhaseResults( + searchPhaseResults, + searchPhaseContext, + SearchPhaseName.DFS_QUERY.getName(), + SearchPhaseName.QUERY.getName() + ); + + assertSame(searchPhaseResults.getAtomicArray(), notTransformedSearchPhaseResult); + } + public void testGetPipelines() { // assertEquals(0, SearchPipelineService.innerGetPipelines(null, "p1").size()); @@ -582,16 +743,23 @@ public void testGetPipelines() { "p2", new BytesArray("{\"response_processors\" : [ { \"fixed_score\": { \"score\" : 2 } } ] }"), XContentType.JSON + ), + "p3", + new PipelineConfiguration( + "p3", + new BytesArray("{\"phase_results_processors\" : [ { \"max_score\" : { } } ]}"), + XContentType.JSON ) ) ); // Return all when no ids specified List pipelines = SearchPipelineService.innerGetPipelines(metadata); - assertEquals(2, pipelines.size()); + assertEquals(3, pipelines.size()); pipelines.sort(Comparator.comparing(PipelineConfiguration::getId)); assertEquals("p1", pipelines.get(0).getId()); assertEquals("p2", pipelines.get(1).getId()); + assertEquals("p3", pipelines.get(2).getId()); // Get specific pipeline pipelines = SearchPipelineService.innerGetPipelines(metadata, "p1"); @@ -607,17 +775,19 @@ public void testGetPipelines() { // Match all pipelines = SearchPipelineService.innerGetPipelines(metadata, "*"); - assertEquals(2, pipelines.size()); + assertEquals(3, pipelines.size()); pipelines.sort(Comparator.comparing(PipelineConfiguration::getId)); assertEquals("p1", pipelines.get(0).getId()); assertEquals("p2", pipelines.get(1).getId()); + assertEquals("p3", pipelines.get(2).getId()); // Match prefix pipelines = SearchPipelineService.innerGetPipelines(metadata, "p*"); - assertEquals(2, pipelines.size()); + assertEquals(3, pipelines.size()); pipelines.sort(Comparator.comparing(PipelineConfiguration::getId)); assertEquals("p1", pipelines.get(0).getId()); assertEquals("p2", pipelines.get(1).getId()); + assertEquals("p3", pipelines.get(2).getId()); } public void testValidatePipeline() throws Exception { @@ -625,6 +795,7 @@ public void testValidatePipeline() throws Exception { ProcessorInfo reqProcessor = new ProcessorInfo("scale_request_size"); ProcessorInfo rspProcessor = new ProcessorInfo("fixed_score"); + ProcessorInfo injProcessor = new ProcessorInfo("max_score"); DiscoveryNode n1 = new DiscoveryNode("n1", buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNode n2 = new DiscoveryNode("n2", buildNewFakeTransportAddress(), Version.CURRENT); PutSearchPipelineRequest putRequest = new PutSearchPipelineRequest( @@ -632,7 +803,8 @@ public void testValidatePipeline() throws Exception { new BytesArray( "{" + "\"request_processors\": [{ \"scale_request_size\": { \"scale\" : 2 } }]," - + "\"response_processors\": [{ \"fixed_score\": { \"score\" : 2 } }]" + + "\"response_processors\": [{ \"fixed_score\": { \"score\" : 2 } }]," + + "\"phase_results_processors\" : [ { \"max_score\" : { } } ]" + "}" ), XContentType.JSON @@ -699,8 +871,7 @@ public void testInlinePipeline() throws Exception { assertEquals(1, pipeline.getSearchResponseProcessors().size()); // Verify that pipeline transforms request - SearchRequest transformedRequest = pipelinedRequest.transformedRequest(); - assertEquals(200, transformedRequest.source().size()); + assertEquals(200, pipelinedRequest.source().size()); int size = 10; SearchHit[] hits = new SearchHit[size]; @@ -730,7 +901,7 @@ public void testExceptionOnPipelineCreation() { "bad_factory", (pf, t, f, c) -> { throw new RuntimeException(); } ); - SearchPipelineService searchPipelineService = createWithProcessors(badFactory, Collections.emptyMap()); + SearchPipelineService searchPipelineService = createWithProcessors(badFactory, Collections.emptyMap(), Collections.emptyMap()); Map pipelineSourceMap = new HashMap<>(); pipelineSourceMap.put(Pipeline.REQUEST_PROCESSORS_KEY, List.of(Map.of("bad_factory", Collections.emptyMap()))); @@ -752,7 +923,11 @@ public void testExceptionOnRequestProcessing() { (pf, t, f, c) -> throwingRequestProcessor ); - SearchPipelineService searchPipelineService = createWithProcessors(throwingRequestProcessorFactory, Collections.emptyMap()); + SearchPipelineService searchPipelineService = createWithProcessors( + throwingRequestProcessorFactory, + Collections.emptyMap(), + Collections.emptyMap() + ); Map pipelineSourceMap = new HashMap<>(); pipelineSourceMap.put(Pipeline.REQUEST_PROCESSORS_KEY, List.of(Map.of("throwing_request", Collections.emptyMap()))); @@ -773,7 +948,11 @@ public void testExceptionOnResponseProcessing() throws Exception { (pf, t, f, c) -> throwingResponseProcessor ); - SearchPipelineService searchPipelineService = createWithProcessors(Collections.emptyMap(), throwingResponseProcessorFactory); + SearchPipelineService searchPipelineService = createWithProcessors( + Collections.emptyMap(), + throwingResponseProcessorFactory, + Collections.emptyMap() + ); Map pipelineSourceMap = new HashMap<>(); pipelineSourceMap.put(Pipeline.RESPONSE_PROCESSORS_KEY, List.of(Map.of("throwing_response", Collections.emptyMap()))); @@ -807,7 +986,7 @@ public void testStats() throws Exception { "throwing_response", (pf, t, f, c) -> throwingResponseProcessor ); - SearchPipelineService searchPipelineService = createWithProcessors(requestProcessors, responseProcessors); + SearchPipelineService searchPipelineService = createWithProcessors(requestProcessors, responseProcessors, Collections.emptyMap()); SearchPipelineMetadata metadata = new SearchPipelineMetadata( Map.of( From d5482f77679da2a8f72a7164e04b1c3d3370c90a Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:04:11 +0530 Subject: [PATCH 026/186] Handle case where IndexShard is null due to failed creation of index (#8237) Signed-off-by: Gaurav Bafna --- server/src/main/java/org/opensearch/index/IndexService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 73797106bb66f..e6930b41088e2 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -603,7 +603,10 @@ public synchronized void removeShard(int shardId, String reason) { private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store store, IndexEventListener listener) { final int shardId = sId.id(); final Settings indexSettings = this.getIndexSettings().getSettings(); - Store remoteStore = indexShard.remoteStore(); + Store remoteStore = null; + if (indexShard != null) { + remoteStore = indexShard.remoteStore(); + } if (store != null) { store.beforeClose(); } From 544b1cac97a848479f2355e11e1a2c18b40e8201 Mon Sep 17 00:00:00 2001 From: Rishab Nahata Date: Thu, 29 Jun 2023 10:44:02 +0530 Subject: [PATCH 027/186] Check UTF16 string size before converting to String to avoid OOME (#7963) * Check UTF16 string size before converting to string to avoid OOME Signed-off-by: Rishab Nahata --- CHANGELOG.md | 2 + .../common/bytes/AbstractBytesReference.java | 14 ++++++- .../bytes/AbstractBytesReferenceTestCase.java | 39 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b2296475fd1..240c016524752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,8 +139,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Snapshot Interop] Add Changes in Create Snapshot Flow for remote store interoperability. ([#7118](https://github.com/opensearch-project/OpenSearch/pull/7118)) - Allow insecure string settings to warn-log usage and advise to migration of a newer secure variant ([#5496](https://github.com/opensearch-project/OpenSearch/pull/5496)) - Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652)) +- Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963)) - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) + ### Deprecated ### Removed diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java b/libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java index 7b3c71321e4f0..043d45223498e 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java +++ b/libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java @@ -33,6 +33,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; +import org.apache.lucene.util.UnicodeUtil; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; @@ -49,6 +50,7 @@ public abstract class AbstractBytesReference implements BytesReference { private Integer hash = null; // we cache the hash of this reference since it can be quite costly to re-calculated it + private static final int MAX_UTF16_LENGTH = Integer.MAX_VALUE >> 1; @Override public int getInt(int index) { @@ -80,9 +82,19 @@ public void writeTo(OutputStream os) throws IOException { } } + protected int getMaxUTF16Length() { + return MAX_UTF16_LENGTH; + } + @Override public String utf8ToString() { - return toBytesRef().utf8ToString(); + BytesRef bytesRef = toBytesRef(); + final char[] ref = new char[bytesRef.length]; + final int len = UnicodeUtil.UTF8toUTF16(bytesRef, ref); + if (len > getMaxUTF16Length()) { + throw new IllegalArgumentException("UTF16 String size is " + len + ", should be less than " + getMaxUTF16Length()); + } + return new String(ref, 0, len); } @Override diff --git a/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java b/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java index dca46c37ca7d2..dd71711f9154c 100644 --- a/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java @@ -478,6 +478,45 @@ public void testToUtf8() throws IOException { // TODO: good way to test? } + public void testUTF8toString_ExceedsMaxLength() { + AbstractBytesReference abr = new TestAbstractBytesReference(); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, abr::utf8ToString); + assertTrue(e.getMessage().contains("UTF16 String size is")); + assertTrue(e.getMessage().contains("should be less than")); + } + + static class TestAbstractBytesReference extends AbstractBytesReference { + @Override + public byte get(int index) { + return 0; + } + + @Override + public int length() { + return 0; + } + + @Override + public BytesReference slice(int from, int length) { + return null; + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public BytesRef toBytesRef() { + return new BytesRef("UTF16 length exceed test"); + } + + @Override + public int getMaxUTF16Length() { + return 1; + } + } + public void testToBytesRef() throws IOException { int length = randomIntBetween(0, PAGE_SIZE); BytesReference pbr = newBytesReference(length); From 87a833fbedfbcf9029d5e14e0597f8c5e8968086 Mon Sep 17 00:00:00 2001 From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Date: Thu, 29 Jun 2023 17:14:30 +0530 Subject: [PATCH 028/186] Enable Point based optimization for custom comparators (#8168) * Enable Point based optimization for custom comparators Signed-off-by: gashutos * Simplifying enableSkipping logic in IndexField Signed-off-by: gashutos * Empty commit Signed-off-by: gashutos * Update server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java Co-authored-by: Andrew Ross Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> --------- Signed-off-by: gashutos Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Co-authored-by: Andrew Ross --- CHANGELOG.md | 1 + .../index/fielddata/IndexFieldData.java | 6 +++++ .../fielddata/IndexNumericFieldData.java | 25 +++++++++++++------ .../DoubleValuesComparatorSource.java | 2 +- .../FloatValuesComparatorSource.java | 2 +- .../IntValuesComparatorSource.java | 2 +- .../LongValuesComparatorSource.java | 2 +- .../UnsignedLongValuesComparatorSource.java | 2 +- 8 files changed, 30 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 240c016524752..e1859437d1ee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038)) - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) +- Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index 529405c90c9a4..36e6a242ecdec 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -120,11 +120,13 @@ abstract class XFieldComparatorSource extends FieldComparatorSource { protected final MultiValueMode sortMode; protected final Object missingValue; protected final Nested nested; + protected boolean enableSkipping; public XFieldComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { this.sortMode = sortMode; this.missingValue = missingValue; this.nested = nested; + this.enableSkipping = true; // true by default } public MultiValueMode sortMode() { @@ -135,6 +137,10 @@ public Nested nested() { return this.nested; } + public void disableSkipping() { + this.enableSkipping = false; + } + /** * Simple wrapper class around a filter that matches parent documents * and a filter that matches child documents. For every root document R, diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java index 052a679de9765..ae8ffd8fe6b97 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java @@ -198,24 +198,35 @@ private XFieldComparatorSource comparatorSource( MultiValueMode sortMode, Nested nested ) { + final XFieldComparatorSource source; switch (targetNumericType) { case HALF_FLOAT: case FLOAT: - return new FloatValuesComparatorSource(this, missingValue, sortMode, nested); + source = new FloatValuesComparatorSource(this, missingValue, sortMode, nested); + break; case DOUBLE: - return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); + source = new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); + break; case UNSIGNED_LONG: - return new UnsignedLongValuesComparatorSource(this, missingValue, sortMode, nested); + source = new UnsignedLongValuesComparatorSource(this, missingValue, sortMode, nested); + break; case DATE: - return dateComparatorSource(missingValue, sortMode, nested); + source = dateComparatorSource(missingValue, sortMode, nested); + break; case DATE_NANOSECONDS: - return dateNanosComparatorSource(missingValue, sortMode, nested); + source = dateNanosComparatorSource(missingValue, sortMode, nested); + break; case LONG: - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); + source = new LongValuesComparatorSource(this, missingValue, sortMode, nested); + break; default: assert !targetNumericType.isFloatingPoint(); - return new IntValuesComparatorSource(this, missingValue, sortMode, nested); + source = new IntValuesComparatorSource(this, missingValue, sortMode, nested); } + if (targetNumericType != getNumericType()) { + source.disableSkipping(); // disable skipping logic for caste of sort field + } + return source; } protected XFieldComparatorSource dateComparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index f8af86c904f2c..34e86070054c9 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -104,7 +104,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final double dMissingValue = (Double) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new DoubleComparator(numHits, null, null, reversed, false) { + return new DoubleComparator(numHits, fieldname, null, reversed, enableSkipping && this.enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 686bef479c179..04a34cd418520 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -97,7 +97,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final float fMissingValue = (Float) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new FloatComparator(numHits, null, null, reversed, false) { + return new FloatComparator(numHits, fieldname, null, reversed, enableSkipping && this.enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new FloatLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java index 90afa5fc64c29..d5ea1eaf7263d 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java @@ -76,7 +76,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final int iMissingValue = (Integer) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new IntComparator(numHits, null, null, reversed, false) { + return new IntComparator(numHits, fieldname, null, reversed, enableSkipping && this.enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new IntLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 462092dca5110..43e033dd59716 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -120,7 +120,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final long lMissingValue = (Long) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new LongComparator(numHits, null, null, reversed, false) { + return new LongComparator(numHits, fieldname, null, reversed, enableSkipping && this.enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index d8b2e9528d2cf..be56b50179114 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -92,7 +92,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final BigInteger ulMissingValue = (BigInteger) missingObject(missingValue, reversed); - return new UnsignedLongComparator(numHits, null, null, reversed, false) { + return new UnsignedLongComparator(numHits, fieldname, null, reversed, enableSkipping && this.enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new UnsignedLongLeafComparator(context) { From 73b7a85d0a41580870a392aac9d47192adefa37a Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Thu, 29 Jun 2023 08:32:07 -0700 Subject: [PATCH 029/186] [Fix] Replace cached time with system clock in MasterService debug logs (#7902) * Replace cached time with system clock in MasterService debug logs Signed-off-by: Sandesh Kumar * Supply System.nanaoTime via TimeSupplier Signed-off-by: Sandesh Kumar * Add absolute time fetch method in Threadpool Signed-off-by: Sandesh Kumar * Rename absoluteTimeInNanos to preciseRelativeTimeInNanos Signed-off-by: Sandesh Kumar --------- Signed-off-by: Sandesh Kumar --- CHANGELOG.md | 1 + .../cluster/service/MasterService.java | 12 ++++---- .../org/opensearch/threadpool/ThreadPool.java | 13 +++++++++ .../cluster/service/MasterServiceTests.java | 28 +++++++++---------- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1859437d1ee3..756f8a4652065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,6 +156,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125)) - Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591)) - Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318)) +- Precise system clock time in MasterService debug logs ([#7902](https://github.com/opensearch-project/OpenSearch/pull/7902)) ### Security diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java index 9712fdbfbe8ec..ffc6f81490c27 100644 --- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java @@ -291,14 +291,14 @@ private void runTasks(TaskInputs taskInputs) { return; } - final long computationStartTime = threadPool.relativeTimeInMillis(); + final long computationStartTime = threadPool.preciseRelativeTimeInNanos(); final TaskOutputs taskOutputs = calculateTaskOutputs(taskInputs, previousClusterState); taskOutputs.notifyFailedTasks(); final TimeValue computationTime = getTimeSince(computationStartTime); logExecutionTime(computationTime, "compute cluster state update", summary); if (taskOutputs.clusterStateUnchanged()) { - final long notificationStartTime = threadPool.relativeTimeInMillis(); + final long notificationStartTime = threadPool.preciseRelativeTimeInNanos(); taskOutputs.notifySuccessfulTasksOnUnchangedClusterState(); final TimeValue executionTime = getTimeSince(notificationStartTime); logExecutionTime(executionTime, "notify listeners on unchanged cluster state", summary); @@ -309,7 +309,7 @@ private void runTasks(TaskInputs taskInputs) { } else { logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), summary); } - final long publicationStartTime = threadPool.relativeTimeInMillis(); + final long publicationStartTime = threadPool.preciseRelativeTimeInNanos(); try { ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(summary, newClusterState, previousClusterState); // new cluster state, notify all listeners @@ -335,8 +335,8 @@ private void runTasks(TaskInputs taskInputs) { } } - private TimeValue getTimeSince(long startTimeMillis) { - return TimeValue.timeValueMillis(Math.max(0, threadPool.relativeTimeInMillis() - startTimeMillis)); + private TimeValue getTimeSince(long startTimeNanos) { + return TimeValue.timeValueMillis(TimeValue.nsecToMSec(threadPool.preciseRelativeTimeInNanos() - startTimeNanos)); } protected void publish(ClusterChangedEvent clusterChangedEvent, TaskOutputs taskOutputs, long startTimeMillis) { @@ -358,7 +358,7 @@ protected boolean blockingAllowed() { } void onPublicationSuccess(ClusterChangedEvent clusterChangedEvent, TaskOutputs taskOutputs) { - final long notificationStartTime = threadPool.relativeTimeInMillis(); + final long notificationStartTime = threadPool.preciseRelativeTimeInNanos(); taskOutputs.processedDifferentClusterState(clusterChangedEvent.previousState(), clusterChangedEvent.state()); try { diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 2c91d5aa33090..ebc68c288e25a 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -330,6 +330,19 @@ public long relativeTimeInNanos() { return cachedTimeThread.relativeTimeInNanos(); } + /** + * Returns a value of nanoseconds that may be used for relative time calculations + * that require the highest precision possible. Performance critical code must use + * either {@link #relativeTimeInNanos()} or {@link #relativeTimeInMillis()} which + * give better performance at the cost of lower precision. + * + * This method should only be used for calculating time deltas. For an epoch based + * timestamp, see {@link #absoluteTimeInMillis()}. + */ + public long preciseRelativeTimeInNanos() { + return System.nanoTime(); + } + /** * Returns the value of milliseconds since UNIX epoch. * diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java index fb47cb8e2d65a..3c27748daa87d 100644 --- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java @@ -97,14 +97,14 @@ public class MasterServiceTests extends OpenSearchTestCase { private static ThreadPool threadPool; - private static long relativeTimeInMillis; + private static long timeDiffInMillis; @BeforeClass public static void createThreadPool() { threadPool = new TestThreadPool(MasterServiceTests.class.getName()) { @Override - public long relativeTimeInMillis() { - return relativeTimeInMillis; + public long preciseRelativeTimeInNanos() { + return timeDiffInMillis * TimeValue.NSEC_PER_MSEC; } }; } @@ -119,7 +119,7 @@ public static void stopThreadPool() { @Before public void randomizeCurrentTime() { - relativeTimeInMillis = randomLongBetween(0L, 1L << 62); + timeDiffInMillis = randomLongBetween(0L, 1L << 50); } private ClusterManagerService createClusterManagerService(boolean makeClusterManager) { @@ -426,7 +426,7 @@ public void testClusterStateUpdateLogging() throws Exception { clusterManagerService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += TimeValue.timeValueSeconds(1).millis(); + timeDiffInMillis += TimeValue.timeValueSeconds(1).millis(); return currentState; } @@ -441,7 +441,7 @@ public void onFailure(String source, Exception e) { clusterManagerService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += TimeValue.timeValueSeconds(2).millis(); + timeDiffInMillis += TimeValue.timeValueSeconds(2).millis(); throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task"); } @@ -456,13 +456,13 @@ public void onFailure(String source, Exception e) {} clusterManagerService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += TimeValue.timeValueSeconds(3).millis(); + timeDiffInMillis += TimeValue.timeValueSeconds(3).millis(); return ClusterState.builder(currentState).incrementVersion().build(); } @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - relativeTimeInMillis += TimeValue.timeValueSeconds(4).millis(); + timeDiffInMillis += TimeValue.timeValueSeconds(4).millis(); } @Override @@ -1080,12 +1080,12 @@ public void testLongClusterStateUpdateLogging() throws Exception { final AtomicReference clusterStateRef = new AtomicReference<>(initialClusterState); clusterManagerService.setClusterStatePublisher((event, publishListener, ackListener) -> { if (event.source().contains("test5")) { - relativeTimeInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( + timeDiffInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( Settings.EMPTY ).millis() + randomLongBetween(1, 1000000); } if (event.source().contains("test6")) { - relativeTimeInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( + timeDiffInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( Settings.EMPTY ).millis() + randomLongBetween(1, 1000000); throw new OpenSearchException("simulated error during slow publication which should trigger logging"); @@ -1101,7 +1101,7 @@ public void testLongClusterStateUpdateLogging() throws Exception { clusterManagerService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += randomLongBetween( + timeDiffInMillis += randomLongBetween( 0L, ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() ); @@ -1124,7 +1124,7 @@ public void onFailure(String source, Exception e) { clusterManagerService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( + timeDiffInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( Settings.EMPTY ).millis() + randomLongBetween(1, 1000000); throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task"); @@ -1143,7 +1143,7 @@ public void onFailure(String source, Exception e) { clusterManagerService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( + timeDiffInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( Settings.EMPTY ).millis() + randomLongBetween(1, 1000000); return ClusterState.builder(currentState).incrementVersion().build(); @@ -1162,7 +1162,7 @@ public void onFailure(String source, Exception e) { clusterManagerService.submitStateUpdateTask("test4", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - relativeTimeInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( + timeDiffInMillis += ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get( Settings.EMPTY ).millis() + randomLongBetween(1, 1000000); return currentState; From 22cb650e11e77bc86183d418fc99b00232559677 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Thu, 29 Jun 2023 20:22:22 +0200 Subject: [PATCH 030/186] Fix HDFS fixture by excluding BouncyCastle (#8359) BouncyCastle excluded from the HDFS test fixture Signed-off-by: Andrey Pleskach --- test/fixtures/hdfs-fixture/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 310b088674d5c..b1a87fe6c3112 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -44,6 +44,7 @@ dependencies { exclude module: 'guava' exclude module: 'protobuf-java' exclude group: 'org.codehaus.jackson' + exclude group: "org.bouncycastle" } api "org.codehaus.jettison:jettison:${versions.jettison}" api "org.apache.commons:commons-compress:1.23.0" @@ -51,7 +52,6 @@ dependencies { api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "io.netty:netty-all:${versions.netty}" api 'com.google.code.gson:gson:2.10.1' - api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" @@ -65,4 +65,5 @@ dependencies { api "org.apache.commons:commons-text:1.10.0" api "commons-net:commons-net:3.9.0" runtimeOnly "com.google.guava:guava:${versions.guava}" + } From 51da85c1f48934cd23e2ff7cedd60d25de8cece3 Mon Sep 17 00:00:00 2001 From: Mohit Godwani <81609427+mgodwan@users.noreply.github.com> Date: Fri, 30 Jun 2023 01:13:55 +0530 Subject: [PATCH 031/186] Enable Fast Double Parser in Jackson (#7909) * Enable Fast Double Parser in Jackson Signed-off-by: Mohit Godwani * Use Fast Double Parser in jackson Signed-off-by: Mohit Godwani --------- Signed-off-by: Mohit Godwani Signed-off-by: Mohit Godwani <81609427+mgodwan@users.noreply.github.com> --- CHANGELOG.md | 1 + .../java/org/opensearch/common/xcontent/cbor/CborXContent.java | 2 ++ .../java/org/opensearch/common/xcontent/json/JsonXContent.java | 3 ++- .../org/opensearch/common/xcontent/smile/SmileXContent.java | 2 ++ .../java/org/opensearch/common/xcontent/yaml/YamlXContent.java | 2 ++ 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 756f8a4652065..f75cc6e508e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919)) +- Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909)) ### Deprecated diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java index d7df53e7a0cf5..46891b279ba43 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java @@ -36,6 +36,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; @@ -75,6 +76,7 @@ public static XContentBuilder contentBuilder() throws IOException { cborFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); cborFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); cborFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + cborFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); cborXContent = new CborXContent(); } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java index 8ff8e7730b189..e6c27e4cf3eef 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java @@ -37,7 +37,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; - +import com.fasterxml.jackson.core.StreamReadFeature; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -78,6 +78,7 @@ public static XContentBuilder contentBuilder() throws IOException { jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); jsonFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + jsonFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); jsonXContent = new JsonXContent(); } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java index e0a39df1589a2..eb968556de8c9 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java @@ -36,6 +36,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import org.opensearch.core.xcontent.DeprecationHandler; @@ -77,6 +78,7 @@ public static XContentBuilder contentBuilder() throws IOException { smileFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); smileFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); smileFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + smileFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); smileXContent = new SmileXContent(); } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java index 1e73cb2bd9c5e..bb4fa9a09d448 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java @@ -35,6 +35,7 @@ import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -70,6 +71,7 @@ public static XContentBuilder contentBuilder() throws IOException { yamlFactory = new YAMLFactory(); yamlFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); yamlFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + yamlFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); yamlXContent = new YamlXContent(); } From 7b436fb5f969b5e4ddff385e1dc7bd43ad3cacd8 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 29 Jun 2023 16:52:07 -0400 Subject: [PATCH 032/186] Update search.aggregation/370_multi_terms.yml after backport (#8358) Signed-off-by: Andriy Redko --- .../rest-api-spec/test/search.aggregation/370_multi_terms.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml index eeab8e78bf830..7db5f31d8e761 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml @@ -716,7 +716,7 @@ setup: --- "aggregate over multi-terms test": - skip: - version: "- 2.9.99" + version: "- 2.8.99" reason: "multi_terms aggregation was introduced in 2.1.0, NPE bug checked by this test case will manifest in any version < 3.0" - do: From ed3124bb2eeed17cf0e7da026caa4c040c6b4906 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 29 Jun 2023 15:44:17 -0700 Subject: [PATCH 033/186] Fix primary balance flaky test (#8366) Signed-off-by: Kunal Kotwani --- .../indices/replication/SegmentReplicationAllocationIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java index 9f382d6a2e9ef..cbaf70c325a57 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java @@ -96,7 +96,6 @@ public void testGlobalPrimaryAllocation() throws Exception { * This test in general passes without primary shard balance as well due to nature of allocation algorithm which * assigns all primary shards first followed by replica copies. */ - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7751") public void testPerIndexPrimaryAllocation() throws Exception { internalCluster().startClusterManagerOnlyNode(); final int maxReplicaCount = 2; @@ -234,7 +233,7 @@ private void verifyPerIndexPrimaryBalance() throws Exception { RoutingNodes nodes = currentState.getRoutingNodes(); for (final Map.Entry index : currentState.getRoutingTable().indicesRouting().entrySet()) { final int totalPrimaryShards = index.getValue().primaryShardsActive(); - final int avgPrimaryShardsPerNode = (int) Math.ceil(totalPrimaryShards * 1f / currentState.getRoutingNodes().size()); + final int avgPrimaryShardsPerNode = (int) Math.floor(totalPrimaryShards * 1f / currentState.getRoutingNodes().size()); for (RoutingNode node : nodes) { final int primaryCount = node.shardsWithState(index.getKey(), STARTED) .stream() @@ -250,7 +249,8 @@ private void verifyPerIndexPrimaryBalance() throws Exception { avgPrimaryShardsPerNode ); } - assertTrue(primaryCount <= avgPrimaryShardsPerNode); + // Asserts value is within the variance threshold (-1/+1 of the average value). + assertTrue(avgPrimaryShardsPerNode - 1 <= primaryCount && primaryCount <= avgPrimaryShardsPerNode + 1); } } }, 60, TimeUnit.SECONDS); From 17ee1ce11d7b4382418454b9319679b9a5e4cbe9 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 29 Jun 2023 21:40:17 -0400 Subject: [PATCH 034/186] Fix search.aggregation/60_empty.yml test case (fails on BWC runs) (#8351) Signed-off-by: Andriy Redko --- .../rest-api-spec/test/search.aggregation/60_empty.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_empty.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_empty.yml index 7b374e3f6a409..54f61d46b6f6c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_empty.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_empty.yml @@ -1,5 +1,8 @@ --- "Empty aggs Body": + - skip: + version: "- 2.8.99" + reason: "the fix was introduced in 2.9.0" - do: index: index: test From a68733af8af7747fd0cc6c0d2704f082f172a1d9 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 30 Jun 2023 10:34:22 -0700 Subject: [PATCH 035/186] Add lower and upper bound support for appropriate variance thresholds (#8372) Signed-off-by: Kunal Kotwani --- .../SegmentReplicationAllocationIT.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java index cbaf70c325a57..b6ea3a094f496 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java @@ -233,24 +233,30 @@ private void verifyPerIndexPrimaryBalance() throws Exception { RoutingNodes nodes = currentState.getRoutingNodes(); for (final Map.Entry index : currentState.getRoutingTable().indicesRouting().entrySet()) { final int totalPrimaryShards = index.getValue().primaryShardsActive(); - final int avgPrimaryShardsPerNode = (int) Math.floor(totalPrimaryShards * 1f / currentState.getRoutingNodes().size()); + final int lowerBoundPrimaryShardsPerNode = (int) Math.floor(totalPrimaryShards * 1f / currentState.getRoutingNodes().size()) + - 1; + final int upperBoundPrimaryShardsPerNode = (int) Math.ceil(totalPrimaryShards * 1f / currentState.getRoutingNodes().size()) + + 1; for (RoutingNode node : nodes) { final int primaryCount = node.shardsWithState(index.getKey(), STARTED) .stream() .filter(ShardRouting::primary) .collect(Collectors.toList()) .size(); - if (primaryCount > avgPrimaryShardsPerNode) { - logger.info( - "--> Primary shard balance assertion failure for index {} on node {} {} <= {}", - index.getKey(), - node.node().getName(), - primaryCount, - avgPrimaryShardsPerNode - ); - } // Asserts value is within the variance threshold (-1/+1 of the average value). - assertTrue(avgPrimaryShardsPerNode - 1 <= primaryCount && primaryCount <= avgPrimaryShardsPerNode + 1); + assertTrue( + "--> Primary balance assertion failure for index " + + index + + "on node " + + node.node().getName() + + " " + + lowerBoundPrimaryShardsPerNode + + " <= " + + primaryCount + + " (assigned) <= " + + upperBoundPrimaryShardsPerNode, + lowerBoundPrimaryShardsPerNode <= primaryCount && primaryCount <= upperBoundPrimaryShardsPerNode + ); } } }, 60, TimeUnit.SECONDS); From 665d2379e9981851d40bab62cbefa0f04ff3e372 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 30 Jun 2023 15:56:47 -0700 Subject: [PATCH 036/186] Fix version scipt for higher patch numbers (#8385) Signed-off-by: Kunal Kotwani --- .github/workflows/version.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index eaf4d085c6946..fdf42a9a2731e 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -30,7 +30,11 @@ jobs: CURRENT_VERSION_ARRAY[2]=$((CURRENT_VERSION_ARRAY[2]+1)) NEXT_VERSION=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}") NEXT_VERSION_UNDERSCORE=$(IFS=_ ; echo "V_${CURRENT_VERSION_ARRAY[*]:0:3}") - NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99") + if [[ ${#CURRENT_VERSION_ARRAY[2]} -gt 1 ]]; then + NEXT_VERSION_ID="${CURRENT_VERSION_ARRAY[0]:0:3}0${CURRENT_VERSION_ARRAY[1]:0:3}${CURRENT_VERSION_ARRAY[2]:0:3}99" + else + NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99") + fi echo "TAG=$TAG" >> $GITHUB_ENV echo "BASE=$BASE" >> $GITHUB_ENV echo "BASE_X=$BASE_X" >> $GITHUB_ENV From 240193a8b129aaffffb39a57403264b8c4893efb Mon Sep 17 00:00:00 2001 From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Date: Sun, 2 Jul 2023 07:33:21 +0530 Subject: [PATCH 037/186] Avoid search_after short cutting in case of missing is specified (#8391) Signed-off-by: gashutos --- .../test/search/90_search_after.yml | 44 +++++++++++++++++++ .../org/opensearch/search/SearchService.java | 4 +- .../opensearch/search/SearchServiceTests.java | 17 +++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 65c1527a68b96..55e1566656faf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -160,6 +160,28 @@ - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" } - match: {hits.hits.0.sort: [1571617804828] } + # search_after with the sort with missing + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"timestamp": null} + - do: + search: + index: test + rest_total_hits_as_int: true + body: + "size": 5 + "sort": [ { "timestamp": { "order": "asc", "missing": "_last" } } ] + search_after: [ "2021-10-21 08:30:04.828" ] # making it out of min/max so only missing value hit is qualified + + - match: { hits.total: 3 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0._source.timestamp: null } + --- "date_nanos": - skip: @@ -276,3 +298,25 @@ - match: {hits.hits.0._index: test } - match: {hits.hits.0._source.population: 15223372036854775800 } - match: {hits.hits.0.sort: [15223372036854775800] } + + # search_after with the sort with missing + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"population": null} + - do: + search: + index: test + rest_total_hits_as_int: true + body: + "size": 5 + "sort": [ { "population": { "order": "asc", "missing": "_last" } } ] + search_after: [15223372036854775801] # making it out of min/max so only missing value hit is qualified + + - match: { hits.total: 3 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0._source.population: null } diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 7d67c6c3b45f4..9daad9112e473 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -1550,7 +1550,9 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre } public static boolean canMatchSearchAfter(FieldDoc searchAfter, MinAndMax minMax, FieldSortBuilder primarySortField) { - if (searchAfter != null && minMax != null && primarySortField != null) { + // Check for sort.missing == null, since in case of missing values sort queries, if segment/shard's min/max + // is out of search_after range, it still should be printed and hence we should not skip segment/shard. + if (searchAfter != null && minMax != null && primarySortField != null && primarySortField.missing() == null) { final Object searchAfterPrimary = searchAfter.fields[0]; if (primarySortField.order() == SortOrder.DESC) { if (minMax.compareMin(searchAfterPrimary) > 0) { diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java index 8f8789a3a0323..74ef289c4b75f 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java @@ -1748,4 +1748,21 @@ public void testCanMatchSearchAfterDescEqualMin() throws IOException { primarySort.order(SortOrder.DESC); assertEquals(SearchService.canMatchSearchAfter(searchAfter, minMax, primarySort), true); } + + /** + * Test canMatchSearchAfter with missing value, even if min/max is out of range + * Min = 0L, Max = 9L, search_after = -1L + * Expected result is canMatch = true + */ + public void testCanMatchSearchAfterWithMissing() throws IOException { + FieldDoc searchAfter = new FieldDoc(0, 0, new Long[] { -1L }); + MinAndMax minMax = new MinAndMax(0L, 9L); + FieldSortBuilder primarySort = new FieldSortBuilder("test"); + primarySort.order(SortOrder.DESC); + // Should be false without missing values + assertEquals(SearchService.canMatchSearchAfter(searchAfter, minMax, primarySort), false); + primarySort.missing("_last"); + // Should be true with missing values + assertEquals(SearchService.canMatchSearchAfter(searchAfter, minMax, primarySort), true); + } } From 1d2274045b42e129b8c06748f8185d2431aa5336 Mon Sep 17 00:00:00 2001 From: Dharmesh Date: Mon, 3 Jul 2023 14:22:31 +0530 Subject: [PATCH 038/186] [Remote Store] Moving stale commit deletion to async flow (#8201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Signed-off-by: Dharmesh 💤 --- .../RemoteStoreBaseIntegTestCase.java | 8 ++ .../opensearch/remotestore/RemoteStoreIT.java | 39 ++++++++++ .../opensearch/index/shard/IndexShard.java | 2 +- .../shard/RemoteStoreRefreshListener.java | 13 +--- .../opensearch/index/shard/StoreRecovery.java | 9 ++- .../store/RemoteSegmentStoreDirectory.java | 44 ++++++++++- .../RemoteSegmentStoreDirectoryFactory.java | 8 +- .../main/java/org/opensearch/node/Node.java | 3 +- .../opensearch/index/IndexModuleTests.java | 2 +- ...moteSegmentStoreDirectoryFactoryTests.java | 5 +- .../RemoteSegmentStoreDirectoryTests.java | 73 ++++++++++++++++--- .../snapshots/SnapshotResiliencyTests.java | 2 +- .../index/shard/IndexShardTestCase.java | 2 +- 13 files changed, 175 insertions(+), 35 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index d226d0d757638..336646b35b5a6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -15,6 +15,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchIntegTestCase; @@ -74,6 +75,13 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) { return remoteStoreIndexSettings(numberOfReplicas, 1); } + protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFieldLimit) { + return Settings.builder() + .put(remoteStoreIndexSettings(numberOfReplicas)) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), totalFieldLimit) + .build(); + } + protected Settings remoteTranslogIndexSettings(int numberOfReplicas, int numberOfShards) { return Settings.builder() .put(remoteStoreIndexSettings(numberOfReplicas, numberOfShards)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 70a41d74a57c5..f6ba8cfed00d0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.index.shard.RemoteStoreRefreshListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.Plugin; import org.opensearch.test.InternalTestCluster; @@ -277,4 +278,42 @@ public void testRemoteSegmentCleanup() throws Exception { public void testRemoteTranslogCleanup() throws Exception { verifyRemoteStoreCleanup(true); } + + public void testStaleCommitDeletionWithInvokeFlush() throws Exception { + internalCluster().startDataOnlyNodes(3); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); + int numberOfIterations = randomIntBetween(5, 15); + indexData(numberOfIterations, true); + String indexUUID = client().admin() + .indices() + .prepareGetSettings(INDEX_NAME) + .get() + .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); + Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID, "/0/segments/metadata"); + // Delete is async. + assertBusy(() -> { + int actualFileCount = getFileCount(indexPath); + if (numberOfIterations <= RemoteStoreRefreshListener.LAST_N_METADATA_FILES_TO_KEEP) { + assertEquals(numberOfIterations, actualFileCount); + } else { + // As delete is async its possible that the file gets created before the deletion or after + // deletion. + assertTrue(actualFileCount >= 10 || actualFileCount <= 11); + } + }, 30, TimeUnit.SECONDS); + } + + public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { + internalCluster().startDataOnlyNodes(3); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); + int numberOfIterations = randomIntBetween(5, 15); + indexData(numberOfIterations, false); + String indexUUID = client().admin() + .indices() + .prepareGetSettings(INDEX_NAME) + .get() + .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); + Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID, "/0/segments/metadata"); + assertEquals(1, getFileCount(indexPath)); + } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index d89d51c713d70..01c0a12d463ea 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2632,7 +2632,7 @@ public void restoreFromSnapshotAndRemoteStore( assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource(); StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); - storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, repositoriesService, listener); + storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, repositoriesService, listener, threadPool); } catch (Exception e) { listener.onFailure(e); } diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 7cfaaafcadd39..ddca12d9283f3 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -85,7 +85,7 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres // Visible for testing static final Set EXCLUDE_FILES = Set.of("write.lock"); // Visible for testing - static final int LAST_N_METADATA_FILES_TO_KEEP = 10; + public static final int LAST_N_METADATA_FILES_TO_KEEP = 10; private final IndexShard indexShard; private final Directory storeDirectory; @@ -200,9 +200,8 @@ private synchronized void syncSegments(boolean isRetry) { // if a new segments_N file is present in local that is not uploaded to remote store yet, it // is considered as a first refresh post commit. A cleanup of stale commit files is triggered. // This is done to avoid delete post each refresh. - // Ideally, we want this to be done in async flow. (GitHub issue #4315) if (isRefreshAfterCommit()) { - deleteStaleCommits(); + remoteDirectory.deleteStaleSegmentsAsync(LAST_N_METADATA_FILES_TO_KEEP); } try (GatedCloseable segmentInfosGatedCloseable = indexShard.getSegmentInfosSnapshot()) { @@ -381,14 +380,6 @@ private String getChecksumOfLocalFile(String file) throws IOException { return localSegmentChecksumMap.get(file); } - private void deleteStaleCommits() { - try { - remoteDirectory.deleteStaleSegments(LAST_N_METADATA_FILES_TO_KEEP); - } catch (IOException e) { - logger.info("Exception while deleting stale commits from remote segment store, will retry delete post next commit", e); - } - } - /** * Updates the last refresh time and refresh seq no which is seen by local store. */ diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 119524e8caf8a..da4e9113143af 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -69,6 +69,7 @@ import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.nio.channels.FileChannel; @@ -356,7 +357,8 @@ void recoverFromSnapshotAndRemoteStore( final IndexShard indexShard, Repository repository, RepositoriesService repositoriesService, - ActionListener listener + ActionListener listener, + ThreadPool threadPool ) { try { if (canRecover(indexShard)) { @@ -384,7 +386,10 @@ void recoverFromSnapshotAndRemoteStore( remoteStoreRepository = shallowCopyShardMetadata.getRemoteStoreRepository(); } - RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService); + RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory( + () -> repositoriesService, + threadPool + ); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( remoteStoreRepository, indexUUID, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index addd8a24af9c5..ac129aca8baf7 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -27,6 +27,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.threadpool.ThreadPool; import java.io.FileNotFoundException; import java.io.IOException; @@ -75,6 +76,8 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement private final RemoteStoreLockManager mdLockManager; + private final ThreadPool threadPool; + /** * To prevent explosion of refresh metadata files, we replace refresh files for the given primary term and generation * This is achieved by uploading refresh metadata file with the same UUID suffix. @@ -96,15 +99,23 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectory.class); + /** + * AtomicBoolean that ensures only one staleCommitDeletion activity is scheduled at a time. + * Visible for testing + */ + protected final AtomicBoolean canDeleteStaleCommits = new AtomicBoolean(true); + public RemoteSegmentStoreDirectory( RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory, - RemoteStoreLockManager mdLockManager + RemoteStoreLockManager mdLockManager, + ThreadPool threadPool ) throws IOException { super(remoteDataDirectory); this.remoteDataDirectory = remoteDataDirectory; this.remoteMetadataDirectory = remoteMetadataDirectory; this.mdLockManager = mdLockManager; + this.threadPool = threadPool; init(); } @@ -574,7 +585,7 @@ public Map getSegmentsUploadedToRemoteStore(lon * @param lastNMetadataFilesToKeep number of metadata files to keep * @throws IOException in case of I/O error while reading from / writing to remote segment store */ - public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException { + private void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException { Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix(MetadataFilenameUtils.METADATA_PREFIX); List sortedMetadataFileList = metadataFiles.stream().sorted(METADATA_FILENAME_COMPARATOR).collect(Collectors.toList()); if (sortedMetadataFileList.size() <= lastNMetadataFilesToKeep) { @@ -656,6 +667,33 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException } } + /** + * Delete stale segment and metadata files asynchronously. + * This method calls {@link RemoteSegmentStoreDirectory#deleteStaleSegments(int)} in an async manner. + * @param lastNMetadataFilesToKeep number of metadata files to keep + */ + public void deleteStaleSegmentsAsync(int lastNMetadataFilesToKeep) { + if (canDeleteStaleCommits.compareAndSet(true, false)) { + try { + threadPool.executor(ThreadPool.Names.REMOTE_PURGE).execute(() -> { + try { + deleteStaleSegments(lastNMetadataFilesToKeep); + } catch (Exception e) { + logger.info( + "Exception while deleting stale commits from remote segment store, will retry delete post next commit", + e + ); + } finally { + canDeleteStaleCommits.set(true); + } + }); + } catch (Exception e) { + logger.info("Exception occurred while scheduling deleteStaleCommits", e); + canDeleteStaleCommits.set(true); + } + } + } + /* Tries to delete shard level directory if it is empty Return true if it deleted it successfully @@ -680,7 +718,7 @@ private boolean deleteIfEmpty() throws IOException { } public void close() throws IOException { - deleteStaleSegments(0); + deleteStaleSegmentsAsync(0); deleteIfEmpty(); } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 03995d5913fb3..3bec84f287ce4 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -20,6 +20,7 @@ import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.util.function.Supplier; @@ -34,8 +35,11 @@ public class RemoteSegmentStoreDirectoryFactory implements IndexStorePlugin.Dire private final Supplier repositoriesService; - public RemoteSegmentStoreDirectoryFactory(Supplier repositoriesService) { + private final ThreadPool threadPool; + + public RemoteSegmentStoreDirectoryFactory(Supplier repositoriesService, ThreadPool threadPool) { this.repositoriesService = repositoriesService; + this.threadPool = threadPool; } @Override @@ -62,7 +66,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, String sh shardId ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool); } catch (RepositoryMissingException e) { throw new IllegalArgumentException("Repository should be created before creating index with remote_store enabled setting", e); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 3742c817118da..d3655671b516d 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -717,7 +717,8 @@ protected Node( clusterService.setRerouteService(rerouteService); final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( - repositoriesServiceReference::get + repositoriesServiceReference::get, + threadPool ); final IndicesService indicesService = new IndicesService( diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index d9d87196ca289..32b8fb5a4dc62 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -252,7 +252,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { writableRegistry(), () -> false, null, - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), translogFactorySupplier ); } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index 7a9cbc12d823b..324315505987b 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -24,6 +24,7 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.nio.file.Path; @@ -41,14 +42,16 @@ public class RemoteSegmentStoreDirectoryFactoryTests extends OpenSearchTestCase private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; + private ThreadPool threadPool; private RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory; @Before public void setup() { repositoriesServiceSupplier = mock(Supplier.class); repositoriesService = mock(RepositoriesService.class); + threadPool = mock(ThreadPool.class); when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); - remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(repositoriesServiceSupplier); + remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(repositoriesServiceSupplier, threadPool); } public void testNewDirectory() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 3417e7b0aee04..66e4b9a357b85 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -30,12 +30,14 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -45,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutorService; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -56,6 +59,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.doReturn; +import static org.hamcrest.CoreMatchers.is; public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase { private RemoteDirectory remoteDataDirectory; @@ -65,21 +69,31 @@ public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase { private RemoteSegmentStoreDirectory remoteSegmentStoreDirectory; private IndexShard indexShard; private SegmentInfos segmentInfos; + private ThreadPool threadPool; @Before public void setup() throws IOException { remoteDataDirectory = mock(RemoteDirectory.class); remoteMetadataDirectory = mock(RemoteDirectory.class); mdLockManager = mock(RemoteStoreMetadataLockManager.class); + threadPool = mock(ThreadPool.class); - remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory(remoteDataDirectory, remoteMetadataDirectory, mdLockManager); + remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDataDirectory, + remoteMetadataDirectory, + mdLockManager, + threadPool + ); Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT).build(); + ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService(); indexShard = newStartedShard(false, indexSettings, new NRTReplicationEngineFactory()); try (Store store = indexShard.store()) { segmentInfos = store.readLastCommittedSegmentsInfo(); } + + when(threadPool.executor(ThreadPool.Names.REMOTE_PURGE)).thenReturn(executorService); } @After @@ -766,41 +780,76 @@ public void testIncorrectChecksumCorruptIndexException() throws IOException { assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } - public void testDeleteStaleCommitsException() throws IOException { + public void testDeleteStaleCommitsException() throws Exception { + populateMetadata(); when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenThrow( new IOException("Error reading") ); - assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.deleteStaleSegments(5)); + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here to validate that in case of exception deleteFile is not + // invoked + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); + + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory, times(0)).deleteFile(any(String.class)); + } + + public void testDeleteStaleCommitsExceptionWhileScheduling() throws Exception { + populateMetadata(); + doThrow(new IllegalArgumentException()).when(threadPool).executor(any(String.class)); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here to validate that in case of exception deleteFile is not + // invoked + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); + + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory, times(0)).deleteFile(any(String.class)); + } + + public void testDeleteStaleCommitsWithDeletionAlreadyInProgress() throws Exception { + populateMetadata(); + remoteSegmentStoreDirectory.canDeleteStaleCommits.set(false); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here to validate that in case of exception deleteFile is not + // invoked + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); + + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(false))); + verify(remoteMetadataDirectory, times(0)).deleteFile(any(String.class)); } - public void testDeleteStaleCommitsWithinThreshold() throws IOException { + public void testDeleteStaleCommitsWithinThreshold() throws Exception { populateMetadata(); // popluateMetadata() adds stub to return 3 metadata files // We are passing lastNMetadataFilesToKeep=5 here so that none of the metadata files will be deleted - remoteSegmentStoreDirectory.deleteStaleSegments(5); + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(5); + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); verify(remoteMetadataDirectory, times(0)).openInput(any(String.class), eq(IOContext.DEFAULT)); } - public void testDeleteStaleCommitsActualDelete() throws IOException { + public void testDeleteStaleCommitsActualDelete() throws Exception { Map> metadataFilenameContentMapping = populateMetadata(); remoteSegmentStoreDirectory.init(); // popluateMetadata() adds stub to return 3 metadata files // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted - remoteSegmentStoreDirectory.deleteStaleSegments(2); + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); for (String metadata : metadataFilenameContentMapping.get("metadata__1__5__abc").values()) { String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; verify(remoteDataDirectory).deleteFile(uploadedFilename); } ; + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); verify(remoteMetadataDirectory).deleteFile("metadata__1__5__abc"); } - public void testDeleteStaleCommitsActualDeleteIOException() throws IOException { + public void testDeleteStaleCommitsActualDeleteIOException() throws Exception { Map> metadataFilenameContentMapping = populateMetadata(); remoteSegmentStoreDirectory.init(); @@ -813,17 +862,18 @@ public void testDeleteStaleCommitsActualDeleteIOException() throws IOException { doThrow(new IOException("Error")).when(remoteDataDirectory).deleteFile(segmentFileWithException); // popluateMetadata() adds stub to return 3 metadata files // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted - remoteSegmentStoreDirectory.deleteStaleSegments(2); + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); for (String metadata : metadataFilenameContentMapping.get("metadata__1__5__abc").values()) { String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; verify(remoteDataDirectory).deleteFile(uploadedFilename); } ; + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); verify(remoteMetadataDirectory, times(0)).deleteFile("metadata__1__5__abc"); } - public void testDeleteStaleCommitsActualDeleteNoSuchFileException() throws IOException { + public void testDeleteStaleCommitsActualDeleteNoSuchFileException() throws Exception { Map> metadataFilenameContentMapping = populateMetadata(); remoteSegmentStoreDirectory.init(); @@ -836,13 +886,14 @@ public void testDeleteStaleCommitsActualDeleteNoSuchFileException() throws IOExc doThrow(new NoSuchFileException(segmentFileWithException)).when(remoteDataDirectory).deleteFile(segmentFileWithException); // popluateMetadata() adds stub to return 3 metadata files // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted - remoteSegmentStoreDirectory.deleteStaleSegments(2); + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); for (String metadata : metadataFilenameContentMapping.get("metadata__1__5__abc").values()) { String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; verify(remoteDataDirectory).deleteFile(uploadedFilename); } ; + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); verify(remoteMetadataDirectory).deleteFile("metadata__1__5__abc"); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 0bb2b604e8f1a..88899a1b282af 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -1835,7 +1835,7 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), repositoriesServiceReference::get, fileCacheCleaner ); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index ea9e9342673db..7f3819563dcbd 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -666,7 +666,7 @@ protected RemoteSegmentStoreDirectory createRemoteSegmentStoreDirectory(ShardId RemoteStoreLockManager remoteStoreLockManager = new RemoteStoreMetadataLockManager( new RemoteBufferedOutputDirectory(getBlobContainer(remoteShardPath.resolveIndex())) ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager, threadPool); } private RemoteDirectory newRemoteDirectory(Path f) throws IOException { From 2c5a22d167460a0a5d2ad61f16914ca309efe427 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 3 Jul 2023 21:55:48 +0530 Subject: [PATCH 039/186] [Remote Store] Add method in BlobContainer to fetch the keys in sorted order (#8314) --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../repositories/s3/S3BlobContainer.java | 51 +++++- .../s3/S3BlobStoreContainerTests.java | 160 ++++++++++++++++-- .../org/opensearch/backwards/IndexingIT.java | 4 - .../common/blobstore/BlobContainer.java | 47 +++++ .../blobstore/fs/FsBlobContainerTests.java | 80 +++++++++ 5 files changed, 320 insertions(+), 22 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index cf749eeffd903..49ebce77a59ad 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ExceptionsHelper; +import org.opensearch.action.ActionListener; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; import org.opensearch.common.blobstore.BlobContainer; @@ -296,6 +297,35 @@ private static DeleteObjectsRequest bulkDelete(String bucket, List blobs .build(); } + @Override + public void listBlobsByPrefixInSortedOrder( + String blobNamePrefix, + int limit, + BlobNameSortOrder blobNameSortOrder, + ActionListener> listener + ) { + // As AWS S3 returns list of keys in Lexicographic order, we don't have to fetch all the keys in order to sort them + // We fetch only keys as per the given limit to optimize the fetch. If provided sort order is not Lexicographic, + // we fall-back to default implementation of fetching all the keys and sorting them. + if (blobNameSortOrder != BlobNameSortOrder.LEXICOGRAPHIC) { + super.listBlobsByPrefixInSortedOrder(blobNamePrefix, limit, blobNameSortOrder, listener); + } else { + if (limit < 0) { + throw new IllegalArgumentException("limit should not be a negative value"); + } + String prefix = blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix); + try (AmazonS3Reference clientReference = blobStore.clientReference()) { + List blobs = executeListing(clientReference, listObjectsRequest(prefix, limit), limit).stream() + .flatMap(listing -> listing.contents().stream()) + .map(s3Object -> new PlainBlobMetadata(s3Object.key().substring(keyPath.length()), s3Object.size())) + .collect(Collectors.toList()); + listener.onResponse(blobs.subList(0, Math.min(limit, blobs.size()))); + } catch (final Exception e) { + listener.onFailure(new IOException("Exception when listing blobs by prefix [" + prefix + "]", e)); + } + } + } + @Override public Map listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException { String prefix = blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix); @@ -339,10 +369,25 @@ public Map children() throws IOException { } private static List executeListing(AmazonS3Reference clientReference, ListObjectsV2Request listObjectsRequest) { + return executeListing(clientReference, listObjectsRequest, -1); + } + + private static List executeListing( + AmazonS3Reference clientReference, + ListObjectsV2Request listObjectsRequest, + int limit + ) { return SocketAccess.doPrivileged(() -> { final List results = new ArrayList<>(); + int totalObjects = 0; ListObjectsV2Iterable listObjectsIterable = clientReference.get().listObjectsV2Paginator(listObjectsRequest); - listObjectsIterable.forEach(results::add); + for (ListObjectsV2Response listObjectsV2Response : listObjectsIterable) { + results.add(listObjectsV2Response); + totalObjects += listObjectsV2Response.contents().size(); + if (limit != -1 && totalObjects > limit) { + break; + } + } return results; }); } @@ -356,6 +401,10 @@ private ListObjectsV2Request listObjectsRequest(String keyPath) { .build(); } + private ListObjectsV2Request listObjectsRequest(String keyPath, int limit) { + return listObjectsRequest(keyPath).toBuilder().maxKeys(Math.min(limit, 1000)).build(); + } + private String buildKey(String blobName) { return keyPath + blobName; } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index ec16f216f1777..a2a7ca8d8bdd5 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -33,6 +33,9 @@ package org.opensearch.repositories.s3; import org.mockito.ArgumentCaptor; +import org.opensearch.action.ActionListener; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.blobstore.DeleteResult; @@ -74,9 +77,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -84,12 +91,12 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; public class S3BlobStoreContainerTests extends OpenSearchTestCase { @@ -187,26 +194,34 @@ private static class MockListObjectsV2ResponseIterator implements Iterator keysListed = new ArrayList<>(); + private final List keysListed; private final boolean throwExceptionOnNextInvocation; public MockListObjectsV2ResponseIterator(int totalPageCount, int s3ObjectsPerPage, long s3ObjectSize) { - this.totalPageCount = totalPageCount; - this.s3ObjectsPerPage = s3ObjectsPerPage; - this.s3ObjectSize = s3ObjectSize; - this.throwExceptionOnNextInvocation = false; + this(totalPageCount, s3ObjectsPerPage, s3ObjectSize, ""); + } + + public MockListObjectsV2ResponseIterator(int totalPageCount, int s3ObjectsPerPage, long s3ObjectSize, String blobPath) { + this(totalPageCount, s3ObjectsPerPage, s3ObjectSize, blobPath, false); } public MockListObjectsV2ResponseIterator( int totalPageCount, int s3ObjectsPerPage, long s3ObjectSize, + String blobPath, boolean throwExceptionOnNextInvocation ) { this.totalPageCount = totalPageCount; this.s3ObjectsPerPage = s3ObjectsPerPage; this.s3ObjectSize = s3ObjectSize; this.throwExceptionOnNextInvocation = throwExceptionOnNextInvocation; + keysListed = new ArrayList<>(); + for (int i = 0; i < totalPageCount * s3ObjectsPerPage; i++) { + keysListed.add(blobPath + UUID.randomUUID().toString()); + } + // S3 lists keys in lexicographic order + keysListed.sort(String::compareTo); } @Override @@ -220,11 +235,12 @@ public ListObjectsV2Response next() { throw SdkException.builder().build(); } if (currInvocationCount.getAndIncrement() < totalPageCount) { - String s3ObjectKey = UUID.randomUUID().toString(); - keysListed.add(s3ObjectKey); - return ListObjectsV2Response.builder() - .contents(Collections.nCopies(s3ObjectsPerPage, S3Object.builder().key(s3ObjectKey).size(s3ObjectSize).build())) - .build(); + List s3Objects = new ArrayList<>(); + for (int i = 0; i < s3ObjectsPerPage; i++) { + String s3ObjectKey = keysListed.get((currInvocationCount.get() - 1) * s3ObjectsPerPage + i); + s3Objects.add(S3Object.builder().key(s3ObjectKey).size(s3ObjectSize).build()); + } + return ListObjectsV2Response.builder().contents(s3Objects).build(); } throw new NoSuchElementException(); } @@ -232,6 +248,10 @@ public ListObjectsV2Response next() { public List getKeysListed() { return keysListed; } + + public int numberOfPagesFetched() { + return currInvocationCount.get(); + } } public void testDelete() throws IOException { @@ -273,10 +293,8 @@ public void testDelete() throws IOException { // keysDeleted will have blobPath also assertEquals(listObjectsV2ResponseIterator.getKeysListed().size(), keysDeleted.size() - 1); assertTrue(keysDeleted.contains(blobPath.buildAsString())); - assertArrayEquals( - listObjectsV2ResponseIterator.getKeysListed().toArray(String[]::new), - keysDeleted.stream().filter(key -> !blobPath.buildAsString().equals(key)).toArray(String[]::new) - ); + keysDeleted.remove(blobPath.buildAsString()); + assertEquals(new HashSet<>(listObjectsV2ResponseIterator.getKeysListed()), new HashSet<>(keysDeleted)); } public void testDeleteItemLevelErrorsDuringDelete() { @@ -772,4 +790,112 @@ private static void assertNumberOfMultiparts(final int expectedParts, final long assertEquals("Expected number of parts [" + expectedParts + "] but got [" + result.v1() + "]", expectedParts, (long) result.v1()); assertEquals("Expected remaining [" + expectedRemaining + "] but got [" + result.v2() + "]", expectedRemaining, (long) result.v2()); } + + public void testListBlobsByPrefix() throws IOException { + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + + final S3Client client = mock(S3Client.class); + final AmazonS3Reference clientReference = new AmazonS3Reference(client); + when(blobStore.clientReference()).thenReturn(clientReference); + + BlobPath blobPath = mock(BlobPath.class); + when(blobPath.buildAsString()).thenReturn("/dummy/path"); + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + final ListObjectsV2Iterable listObjectsV2Iterable = mock(ListObjectsV2Iterable.class); + when(client.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Iterable); + + MockListObjectsV2ResponseIterator iterator = new MockListObjectsV2ResponseIterator(2, 5, 100); + when(listObjectsV2Iterable.iterator()).thenReturn(iterator); + + Map listOfBlobs = blobContainer.listBlobsByPrefix(null); + assertEquals(10, listOfBlobs.size()); + + Set keys = iterator.keysListed.stream() + .map(s -> s.substring(blobPath.buildAsString().length())) + .collect(Collectors.toSet()); + assertEquals(keys, listOfBlobs.keySet()); + } + + private void testListBlobsByPrefixInLexicographicOrder( + int limit, + int expectedNumberofPagesFetched, + BlobContainer.BlobNameSortOrder blobNameSortOrder + ) throws IOException { + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + + final S3Client client = mock(S3Client.class); + final AmazonS3Reference clientReference = new AmazonS3Reference(client); + when(blobStore.clientReference()).thenReturn(clientReference); + + BlobPath blobPath = mock(BlobPath.class); + when(blobPath.buildAsString()).thenReturn("/dummy/path"); + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + final ListObjectsV2Iterable listObjectsV2Iterable = mock(ListObjectsV2Iterable.class); + when(client.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Iterable); + + final MockListObjectsV2ResponseIterator iterator = new MockListObjectsV2ResponseIterator(2, 5, 100, blobPath.buildAsString()); + when(listObjectsV2Iterable.iterator()).thenReturn(iterator); + + if (limit >= 0) { + blobContainer.listBlobsByPrefixInSortedOrder(null, limit, blobNameSortOrder, new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + int actualLimit = Math.max(0, Math.min(limit, 10)); + assertEquals(actualLimit, blobMetadata.size()); + + List keys = iterator.keysListed.stream() + .map(s -> s.substring(blobPath.buildAsString().length())) + .collect(Collectors.toList()); + Comparator keysComparator = String::compareTo; + if (blobNameSortOrder != BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC) { + keysComparator = Collections.reverseOrder(String::compareTo); + } + keys.sort(keysComparator); + List sortedKeys = keys.subList(0, actualLimit); + assertEquals(sortedKeys, blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList())); + assertEquals(expectedNumberofPagesFetched, iterator.numberOfPagesFetched()); + } + + @Override + public void onFailure(Exception e) { + fail("blobContainer.listBlobsByPrefixInLexicographicOrder failed with exception: " + e.getMessage()); + } + }); + } else { + assertThrows( + IllegalArgumentException.class, + () -> blobContainer.listBlobsByPrefixInSortedOrder(null, limit, blobNameSortOrder, new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) {} + + @Override + public void onFailure(Exception e) {} + }) + ); + } + } + + public void testListBlobsByPrefixInLexicographicOrderWithNegativeLimit() throws IOException { + testListBlobsByPrefixInLexicographicOrder(-5, 0, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithZeroLimit() throws IOException { + testListBlobsByPrefixInLexicographicOrder(0, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithLimitLessThanPageSize() throws IOException { + testListBlobsByPrefixInLexicographicOrder(2, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithLimitGreaterThanPageSize() throws IOException { + testListBlobsByPrefixInLexicographicOrder(8, 2, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithLimitGreaterThanNumberOfRecords() throws IOException { + testListBlobsByPrefixInLexicographicOrder(12, 2, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } } diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java index b867b90af333c..f1f469544f634 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java @@ -113,8 +113,6 @@ private void printClusterRouting() throws IOException, ParseException { /** * This test verifies that segment replication does not break when primary shards are on lower OS version. It does this * by verifying replica shards contains same number of documents as primary's. - * - * @throws Exception */ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) { @@ -164,8 +162,6 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { * This test creates a cluster with primary on higher version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider}; * replica shard allocation on lower OpenSearch version is prevented. Thus, this test though cover the use case where * primary shard containing nodes are running on higher OS version while replicas are unassigned. - * - * @throws Exception */ public void testIndexingWithReplicaOnBwcNodes() throws Exception { if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) { diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java index ac38768c9f3d3..e626824e7e271 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java @@ -32,10 +32,14 @@ package org.opensearch.common.blobstore; +import org.opensearch.action.ActionListener; + import java.io.IOException; import java.io.InputStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -191,4 +195,47 @@ default long readBlobPreferredLength() { * @throws IOException if there were any failures in reading from the blob container. */ Map listBlobsByPrefix(String blobNamePrefix) throws IOException; + + /** + * The type representing sort order of blob names + */ + enum BlobNameSortOrder { + + LEXICOGRAPHIC(Comparator.comparing(BlobMetadata::name)); + + final Comparator comparator; + + public Comparator comparator() { + return comparator; + } + + BlobNameSortOrder(final Comparator comparator) { + this.comparator = comparator; + } + } + + /** + * Lists all blobs in the container that match the specified prefix in lexicographic order + * @param blobNamePrefix The prefix to match against blob names in the container. + * @param limit Limits the result size to min(limit, number of keys) + * @param blobNameSortOrder Comparator to sort keys with + * @param listener the listener to be notified upon request completion + */ + default void listBlobsByPrefixInSortedOrder( + String blobNamePrefix, + int limit, + BlobNameSortOrder blobNameSortOrder, + ActionListener> listener + ) { + if (limit < 0) { + throw new IllegalArgumentException("limit should not be a negative value"); + } + try { + List blobNames = new ArrayList<>(listBlobsByPrefix(blobNamePrefix).values()); + blobNames.sort(blobNameSortOrder.comparator()); + listener.onResponse(blobNames.subList(0, Math.min(blobNames.size(), limit))); + } catch (Exception e) { + listener.onFailure(e); + } + } } diff --git a/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java index 6c36368bfe446..f139a5d4e3bb1 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java @@ -34,6 +34,9 @@ import org.apache.lucene.tests.mockfile.FilterFileSystemProvider; import org.apache.lucene.tests.mockfile.FilterSeekableByteChannel; import org.apache.lucene.tests.util.LuceneTestCase; +import org.opensearch.action.ActionListener; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; @@ -54,10 +57,14 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.nio.file.spi.FileSystemProvider; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -118,6 +125,79 @@ public void testIsTempBlobName() { assertThat(FsBlobContainer.isTempBlobName(tempBlobName), is(true)); } + private void testListBlobsByPrefixInSortedOrder(int limit, BlobContainer.BlobNameSortOrder blobNameSortOrder) throws IOException { + + final Path path = PathUtils.get(createTempDir().toString()); + + List blobsInFileSystem = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + final String blobName = randomAlphaOfLengthBetween(1, 20).toLowerCase(Locale.ROOT); + final byte[] blobData = randomByteArrayOfLength(randomIntBetween(1, frequently() ? 512 : 1 << 20)); // rarely up to 1mb + Files.write(path.resolve(blobName), blobData); + blobsInFileSystem.add(blobName); + } + + final FsBlobContainer container = new FsBlobContainer( + new FsBlobStore(randomIntBetween(1, 8) * 1024, path, false), + BlobPath.cleanPath(), + path + ); + + if (limit >= 0) { + container.listBlobsByPrefixInSortedOrder(null, limit, blobNameSortOrder, new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + int actualLimit = Math.min(limit, 10); + assertEquals(actualLimit, blobMetadata.size()); + + if (blobNameSortOrder == BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC) { + blobsInFileSystem.sort(String::compareTo); + } else { + blobsInFileSystem.sort(Collections.reverseOrder(String::compareTo)); + } + List keys = blobsInFileSystem.subList(0, actualLimit); + assertEquals(keys, blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList())); + } + + @Override + public void onFailure(Exception e) { + fail("blobContainer.listBlobsByPrefixInLexicographicOrder failed with exception: " + e.getMessage()); + } + }); + } else { + assertThrows( + IllegalArgumentException.class, + () -> container.listBlobsByPrefixInSortedOrder(null, limit, blobNameSortOrder, new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) {} + + @Override + public void onFailure(Exception e) {} + }) + ); + } + } + + public void testListBlobsByPrefixInLexicographicOrderWithNegativeLimit() throws IOException { + testListBlobsByPrefixInSortedOrder(-5, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithZeroLimit() throws IOException { + testListBlobsByPrefixInSortedOrder(0, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithLimitLessThanNumberOfRecords() throws IOException { + testListBlobsByPrefixInSortedOrder(8, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithLimitNumberOfRecords() throws IOException { + testListBlobsByPrefixInSortedOrder(10, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + + public void testListBlobsByPrefixInLexicographicOrderWithLimitGreaterThanNumberOfRecords() throws IOException { + testListBlobsByPrefixInSortedOrder(12, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + } + static class MockFileSystemProvider extends FilterFileSystemProvider { final Consumer onRead; From a3b515aaa7e14a18f98aef80658a630bb62bb4cc Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 4 Jul 2023 09:29:47 +0530 Subject: [PATCH 040/186] [Remote Store] Add remote store utils useful to name metadata files (#8412) Signed-off-by: Sachin Kale --- .../index/remote/RemoteStoreUtils.java | 51 +++++++++++++++++++ .../index/remote/RemoteStoreUtilsTests.java | 41 +++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java create mode 100644 server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java new file mode 100644 index 0000000000000..1eeadfe228a45 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.remote; + +import java.util.Arrays; + +/** + * Utils for remote store + * + * @opensearch.internal + */ +public class RemoteStoreUtils { + public static final int LONG_MAX_LENGTH = String.valueOf(Long.MAX_VALUE).length(); + + /** + * This method subtracts given numbers from Long.MAX_VALUE and returns a string representation of the result. + * The resultant string is guaranteed to be of the same length that of Long.MAX_VALUE. If shorter, we add left padding + * of 0s to the string. + * @param num number to get the inverted long string for + * @return String value of Long.MAX_VALUE - num + */ + public static String invertLong(long num) { + if (num < 0) { + throw new IllegalArgumentException("Negative long values are not allowed"); + } + String invertedLong = String.valueOf(Long.MAX_VALUE - num); + char[] characterArray = new char[LONG_MAX_LENGTH - invertedLong.length()]; + Arrays.fill(characterArray, '0'); + + return new String(characterArray) + invertedLong; + } + + /** + * This method converts the given string into long and subtracts it from Long.MAX_VALUE + * @param str long in string format to be inverted + * @return long value of the invert result + */ + public static long invertLong(String str) { + long num = Long.parseLong(str); + if (num < 0) { + throw new IllegalArgumentException("Strings representing negative long values are not allowed"); + } + return Long.MAX_VALUE - num; + } +} diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java new file mode 100644 index 0000000000000..5b9135afb66f3 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.remote; + +import org.opensearch.test.OpenSearchTestCase; + +public class RemoteStoreUtilsTests extends OpenSearchTestCase { + + public void testInvertToStrInvalid() { + assertThrows(IllegalArgumentException.class, () -> RemoteStoreUtils.invertLong(-1)); + } + + public void testInvertToStrValid() { + assertEquals("9223372036854774573", RemoteStoreUtils.invertLong(1234)); + assertEquals("0000000000000001234", RemoteStoreUtils.invertLong(9223372036854774573L)); + } + + public void testInvertToLongInvalid() { + assertThrows(IllegalArgumentException.class, () -> RemoteStoreUtils.invertLong("-5")); + } + + public void testInvertToLongValid() { + assertEquals(1234, RemoteStoreUtils.invertLong("9223372036854774573")); + assertEquals(9223372036854774573L, RemoteStoreUtils.invertLong("0000000000000001234")); + } + + public void testinvert() { + assertEquals(0, RemoteStoreUtils.invertLong(RemoteStoreUtils.invertLong(0))); + assertEquals(Long.MAX_VALUE, RemoteStoreUtils.invertLong(RemoteStoreUtils.invertLong(Long.MAX_VALUE))); + for (int i = 0; i < 10; i++) { + long num = randomLongBetween(1, Long.MAX_VALUE); + assertEquals(num, RemoteStoreUtils.invertLong(RemoteStoreUtils.invertLong(num))); + } + } +} From 64c0871fb529b17bc2360cf8a700d969ba302806 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 4 Jul 2023 12:30:33 +0530 Subject: [PATCH 041/186] Fix flaky tests in restore flow using snapshot interop (#8422) Signed-off-by: Sachin Kale --- .../snapshots/RestoreSnapshotIT.java | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index 9f492bbaee01a..e362b7f61e8e6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -183,8 +183,8 @@ public void testRestoreRemoteStoreIndicesWithoutRemoteTranslog() throws IOExcept public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnabled) throws IOException, ExecutionException, InterruptedException { - internalCluster().startClusterManagerOnlyNode(); - final String primaryNode = internalCluster().startNode(); + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); String indexName1 = "testindex1"; String indexName2 = "testindex2"; String snapshotRepoName = "test-restore-snapshot-repo"; @@ -216,7 +216,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable indexDocuments(client, indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); - final String secondNode = internalCluster().startNode(); + internalCluster().startDataOnlyNode(); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() @@ -273,10 +273,12 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); // deleting data for restoredIndexName1 and restoring from remote store. - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); ensureRed(restoredIndexName1); - assertAcked(client().admin().indices().prepareClose(restoredIndexName1)); - client().admin() + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); + client.admin() .cluster() .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); ensureYellowAndNoInitializingShards(restoredIndexName1); @@ -300,7 +302,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED); ensureGreen(restoredIndexName1Seg); - GetIndexResponse getIndexResponse = client().admin() + GetIndexResponse getIndexResponse = client.admin() .indices() .getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true)) .get(); @@ -331,7 +333,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED); ensureGreen(restoredIndexName1Doc); - getIndexResponse = client().admin() + getIndexResponse = client.admin() .indices() .getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true)) .get(); @@ -347,8 +349,8 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable } public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startNode(); + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); String indexName1 = "testindex1"; String indexName2 = "testindex2"; String snapshotRepoName = "test-restore-snapshot-repo"; @@ -378,7 +380,7 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { indexDocuments(client, indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); - final String secondNode = internalCluster().startNode(); + internalCluster().startDataOnlyNode(); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() @@ -435,10 +437,12 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); // deleting data for restoredIndexName1 and restoring from remote store. - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(indexName1))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); ensureRed(indexName1); - assertAcked(client().admin().indices().prepareClose(indexName1)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture()); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(indexName1)); + client.admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture()); ensureYellowAndNoInitializingShards(indexName1); ensureGreen(indexName1); assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); @@ -449,8 +453,8 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { } public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException { - internalCluster().startClusterManagerOnlyNode(); - final String primaryNode = internalCluster().startNode(); + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); String indexName1 = "testindex1"; String indexName2 = "testindex2"; String snapshotRepoName = "test-restore-snapshot-repo"; @@ -479,7 +483,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException indexDocuments(client, indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); - final String secondNode = internalCluster().startNode(); + internalCluster().startDataOnlyNode(); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() @@ -513,9 +517,11 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); // deleting data for restoredIndexName1 and restoring from remote store. - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1))); - assertAcked(client().admin().indices().prepareClose(restoredIndexName1)); - client().admin() + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); + client.admin() .cluster() .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); ensureYellowAndNoInitializingShards(restoredIndexName1); From 7a5c810a238e0566f0c7bdb6fbc774dafb3674b0 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Tue, 4 Jul 2023 20:30:14 +0530 Subject: [PATCH 042/186] Adds log4j configuration for telemetry-otel plugin (#8393) * Adds log4j configuration for telemetry-otel plugin Signed-off-by: Gagan Juneja * Updates changelog Signed-off-by: Gagan Juneja * Fixes spotless Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 1 + plugins/telemetry-otel/build.gradle | 22 +++++++++++++++ .../config/telemetry-otel/log4j2.properties | 27 +++++++++++++++++++ .../tracing/OTelResourceProvider.java | 2 +- 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 plugins/telemetry-otel/config/telemetry-otel/log4j2.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index f75cc6e508e7a..562e9da458cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460)) - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230)) - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541)) +- Adds log4j configuration for telemetry LogSpanExporter ([#8393](https://github.com/opensearch-project/OpenSearch/pull/8393)) ### Security diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle index 7a56621be5f1e..2c275388cce38 100644 --- a/plugins/telemetry-otel/build.gradle +++ b/plugins/telemetry-otel/build.gradle @@ -54,3 +54,25 @@ thirdPartyAudit { 'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider' ) } + +tasks.named("bundlePlugin").configure { + from('config/telemetry-otel') { + into 'config' + } +} + +tasks.register("writeTestJavaPolicy") { + doLast { + final File tmp = file("${buildDir}/tmp") + if (tmp.exists() == false && tmp.mkdirs() == false) { + throw new GradleException("failed to create temporary directory [${tmp}]") + } + final File javaPolicy = file("${tmp}/java.policy") + javaPolicy.write( + [ + "grant {", + " permission java.io.FilePermission \"config\", \"read\";", + "};" + ].join("\n")) + } +} diff --git a/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties b/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties new file mode 100644 index 0000000000000..544f42bd5513b --- /dev/null +++ b/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties @@ -0,0 +1,27 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + + +appender.tracing.type = RollingFile +appender.tracing.name = tracing +appender.tracing.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_otel_traces.log +appender.tracing.filePermissions = rw-r----- +appender.tracing.layout.type = PatternLayout +appender.tracing.layout.pattern = %m%n +appender.tracing.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_otel_traces-%i.log.gz +appender.tracing.policies.type = Policies +appender.tracing.policies.size.type = SizeBasedTriggeringPolicy +appender.tracing.policies.size.size = 1GB +appender.tracing.strategy.type = DefaultRolloverStrategy +appender.tracing.strategy.max = 4 + + +logger.exporter.name = io.opentelemetry.exporter.logging.LoggingSpanExporter +logger.exporter.level = INFO +logger.exporter.appenderRef.tracing.ref = tracing +logger.exporter.additivity = false diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java index 04bade9ec942a..292165979c2f2 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java @@ -42,7 +42,7 @@ private OTelResourceProvider() {} public static OpenTelemetry get(Settings settings) { return get( settings, - new LoggingSpanExporter(), + LoggingSpanExporter.create(), ContextPropagators.create(W3CTraceContextPropagator.getInstance()), Sampler.alwaysOn() ); From a0299fceda9900f8bfdbf8b0229ad1e8884a4fd4 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 4 Jul 2023 11:55:39 -0400 Subject: [PATCH 043/186] Support extension additional settings with extension REST initialization (#8414) * Support extension additional settings with extension REST initialization Signed-off-by: Craig Perkins * Add CHANGELOG entry Signed-off-by: Craig Perkins * Add tests Signed-off-by: Craig Perkins * Add addition setting types to test Signed-off-by: Craig Perkins * Address code review feedback Signed-off-by: Craig Perkins * Check for missing values Signed-off-by: Craig Perkins * Use Version.CURRENT Signed-off-by: Craig Perkins * Switch minimum compat version back to 3.0.0 Signed-off-by: Craig Perkins * Remove hardcoded versions Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../extensions/ExtensionDependency.java | 37 ------ .../extensions/ExtensionsManager.java | 6 +- .../rest/RestInitializeExtensionAction.java | 111 +++++++++++----- .../extensions/ExtensionsManagerTests.java | 12 -- .../RestInitializeExtensionActionTests.java | 119 +++++++++++++++++- 6 files changed, 202 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 562e9da458cf5..0a8deb28833e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) - Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) +- [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java b/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java index 1423a30bbe307..56c9f0387b13e 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java @@ -16,10 +16,6 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.core.common.Strings; -import org.opensearch.core.xcontent.XContentParser; - -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; /** * This class handles the dependent extensions information @@ -60,39 +56,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVersion(version); } - public static ExtensionDependency parse(XContentParser parser) throws IOException { - String uniqueId = null; - Version version = null; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - String fieldName = parser.currentName(); - parser.nextToken(); - - switch (fieldName) { - case UNIQUE_ID: - uniqueId = parser.text(); - break; - case VERSION: - try { - version = Version.fromString(parser.text()); - } catch (IllegalArgumentException e) { - throw e; - } - break; - default: - parser.skipChildren(); - break; - } - } - if (Strings.isNullOrEmpty(uniqueId)) { - throw new IOException("Required field [uniqueId] is missing in the request for the dependent extension"); - } else if (version == null) { - throw new IOException("Required field [version] is missing in the request for the dependent extension"); - } - return new ExtensionDependency(uniqueId, version); - - } - /** * The uniqueId of the dependency extension * diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 9987497b5fac0..cb22c8d864b1b 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -105,7 +105,7 @@ public static enum OpenSearchRequestType { /** * Instantiate a new ExtensionsManager object to handle requests and responses from extensions. This is called during Node bootstrap. * - * @param additionalSettings Additional settings to read in from extensions.yml + * @param additionalSettings Additional settings to read in from extension initialization request * @throws IOException If the extensions discovery file is not properly retrieved. */ public ExtensionsManager(Set> additionalSettings) throws IOException { @@ -504,4 +504,8 @@ void setAddSettingsUpdateConsumerRequestHandler(AddSettingsUpdateConsumerRequest Settings getEnvironmentSettings() { return environmentSettings; } + + public Set> getAdditionalSettings() { + return this.additionalSettings; + } } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index e0806f8172278..f47f342617732 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -8,9 +8,14 @@ package org.opensearch.extensions.rest; +import org.opensearch.Version; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.extensions.ExtensionDependency; import org.opensearch.extensions.ExtensionScopedSettings; import org.opensearch.extensions.ExtensionsManager; @@ -23,12 +28,16 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletionException; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.rest.RestRequest.Method.POST; /** @@ -62,36 +71,79 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client String openSearchVersion = null; String minimumCompatibleVersion = null; List dependencies = new ArrayList<>(); + Set additionalSettingsKeys = extensionsManager.getAdditionalSettings() + .stream() + .map(s -> s.getKey()) + .collect(Collectors.toSet()); - try (XContentParser parser = request.contentParser()) { - parser.nextToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - String currentFieldName = parser.currentName(); - parser.nextToken(); - if ("name".equals(currentFieldName)) { - name = parser.text(); - } else if ("uniqueId".equals(currentFieldName)) { - uniqueId = parser.text(); - } else if ("hostAddress".equals(currentFieldName)) { - hostAddress = parser.text(); - } else if ("port".equals(currentFieldName)) { - port = parser.text(); - } else if ("version".equals(currentFieldName)) { - version = parser.text(); - } else if ("opensearchVersion".equals(currentFieldName)) { - openSearchVersion = parser.text(); - } else if ("minimumCompatibleVersion".equals(currentFieldName)) { - minimumCompatibleVersion = parser.text(); - } else if ("dependencies".equals(currentFieldName)) { - ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - dependencies.add(ExtensionDependency.parse(parser)); + Tuple> unreadExtensionTuple = XContentHelper.convertToMap( + request.content(), + false, + request.getXContentType().xContent().mediaType() + ); + Map extensionMap = unreadExtensionTuple.v2(); + + ExtensionScopedSettings extAdditionalSettings = new ExtensionScopedSettings(extensionsManager.getAdditionalSettings()); + + try { + // checking to see whether any required fields are missing from extension initialization request or not + String[] requiredFields = { + "name", + "uniqueId", + "hostAddress", + "port", + "version", + "opensearchVersion", + "minimumCompatibleVersion" }; + List missingFields = Arrays.stream(requiredFields) + .filter(field -> !extensionMap.containsKey(field)) + .collect(Collectors.toList()); + if (!missingFields.isEmpty()) { + throw new IOException("Extension is missing these required fields : " + missingFields); + } + + // Parse extension dependencies + List extensionDependencyList = new ArrayList(); + if (extensionMap.get("dependencies") != null) { + List> extensionDependencies = new ArrayList<>( + (Collection>) extensionMap.get("dependencies") + ); + for (HashMap dependency : extensionDependencies) { + if (Strings.isNullOrEmpty((String) dependency.get("uniqueId"))) { + throw new IOException("Required field [uniqueId] is missing in the request for the dependent extension"); + } else if (dependency.get("version") == null) { + throw new IOException("Required field [version] is missing in the request for the dependent extension"); } + extensionDependencyList.add( + new ExtensionDependency( + dependency.get("uniqueId").toString(), + Version.fromString(dependency.get("version").toString()) + ) + ); } } + + Map additionalSettingsMap = extensionMap.entrySet() + .stream() + .filter(kv -> additionalSettingsKeys.contains(kv.getKey())) + .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); + + Settings.Builder output = Settings.builder(); + output.loadFromMap(additionalSettingsMap); + extAdditionalSettings.applySettings(output.build()); + + // Create extension read from initialization request + name = extensionMap.get("name").toString(); + uniqueId = extensionMap.get("uniqueId").toString(); + hostAddress = extensionMap.get("hostAddress").toString(); + port = extensionMap.get("port").toString(); + version = extensionMap.get("version").toString(); + openSearchVersion = extensionMap.get("opensearchVersion").toString(); + minimumCompatibleVersion = extensionMap.get("minimumCompatibleVersion").toString(); + dependencies = extensionDependencyList; } catch (IOException e) { - throw new IOException("Missing attribute", e); + logger.warn("loading extension has been failed because of exception : " + e.getMessage()); + return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); } Extension extension = new Extension( @@ -103,8 +155,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client openSearchVersion, minimumCompatibleVersion, dependencies, - // TODO add this to the API (https://github.com/opensearch-project/OpenSearch/issues/8032) - new ExtensionScopedSettings(Collections.emptySet()) + extAdditionalSettings ); try { extensionsManager.loadExtension(extension); diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index f8ec138d8eff2..713a70c6a7d3e 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -45,8 +45,6 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterSettingsResponse; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.env.EnvironmentSettingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -398,16 +396,6 @@ public void testExtensionDependency() throws Exception { } } - public void testParseExtensionDependency() throws Exception { - XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"uniqueId\": \"test1\", \"version\": \"2.0.0\"}"); - - assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); - ExtensionDependency dependency = ExtensionDependency.parse(parser); - - assertEquals("test1", dependency.getUniqueId()); - assertEquals(Version.fromString("2.0.0"), dependency.getVersion()); - } - public void testInitialize() throws Exception { ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 8d027b7fca9c2..7dd616c678e74 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -9,24 +9,33 @@ package org.opensearch.extensions.rest; import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import org.junit.After; import org.junit.Before; +import org.mockito.Mockito; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.xcontent.XContentType; import org.opensearch.extensions.ExtensionsManager; +import org.opensearch.extensions.ExtensionsSettings; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; @@ -88,8 +97,12 @@ public void testRestInitializeExtensionActionResponse() throws Exception { ExtensionsManager extensionsManager = mock(ExtensionsManager.class); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(extensionsManager); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," - + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"3.0.0\"," - + "\"minimumCompatibleVersion\":\"3.0.0\"}"; + + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" + + Version.CURRENT.toString() + + "\"," + + "\"minimumCompatibleVersion\":\"" + + Version.CURRENT.minimumCompatibilityVersion().toString() + + "\"}"; RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(content), XContentType.JSON) .withMethod(RestRequest.Method.POST) .build(); @@ -106,8 +119,12 @@ public void testRestInitializeExtensionActionFailure() throws Exception { RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(extensionsManager); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"\",\"hostAddress\":\"127.0.0.1\"," - + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"3.0.0\"," - + "\"minimumCompatibleVersion\":\"3.0.0\"}"; + + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" + + Version.CURRENT.toString() + + "\"," + + "\"minimumCompatibleVersion\":\"" + + Version.CURRENT.minimumCompatibilityVersion().toString() + + "\"}"; RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(content), XContentType.JSON) .withMethod(RestRequest.Method.POST) .build(); @@ -121,4 +138,98 @@ public void testRestInitializeExtensionActionFailure() throws Exception { ); } + public void testRestInitializeExtensionActionResponseWithAdditionalSettings() throws Exception { + Setting boolSetting = Setting.boolSetting("boolSetting", false, Setting.Property.ExtensionScope); + Setting stringSetting = Setting.simpleString("stringSetting", "default", Setting.Property.ExtensionScope); + Setting intSetting = Setting.intSetting("intSetting", 0, Setting.Property.ExtensionScope); + Setting listSetting = Setting.listSetting( + "listSetting", + List.of("first", "second", "third"), + Function.identity(), + Setting.Property.ExtensionScope + ); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager spy = spy(extensionsManager); + + // optionally, you can stub out some methods: + when(spy.getAdditionalSettings()).thenCallRealMethod(); + Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); + Mockito.doNothing().when(spy).initialize(); + RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); + final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" + + Version.CURRENT.toString() + + "\"," + + "\"minimumCompatibleVersion\":\"" + + Version.CURRENT.minimumCompatibilityVersion().toString() + + "\",\"boolSetting\":true,\"stringSetting\":\"customSetting\",\"intSetting\":5,\"listSetting\":[\"one\",\"two\",\"three\"]}"; + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(content), XContentType.JSON) + .withMethod(RestRequest.Method.POST) + .build(); + + FakeRestChannel channel = new FakeRestChannel(request, false, 0); + restInitializeExtensionAction.handleRequest(request, channel, null); + + assertEquals(channel.capturedResponse().status(), RestStatus.ACCEPTED); + assertTrue(channel.capturedResponse().content().utf8ToString().contains("A request to initialize an extension has been sent.")); + + Optional extension = spy.lookupExtensionSettingsById("ad-extension"); + assertTrue(extension.isPresent()); + assertEquals(true, extension.get().getAdditionalSettings().get(boolSetting)); + assertEquals("customSetting", extension.get().getAdditionalSettings().get(stringSetting)); + assertEquals(5, extension.get().getAdditionalSettings().get(intSetting)); + + List listSettingValue = (List) extension.get().getAdditionalSettings().get(listSetting); + assertTrue(listSettingValue.contains("one")); + assertTrue(listSettingValue.contains("two")); + assertTrue(listSettingValue.contains("three")); + } + + public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsingDefault() throws Exception { + Setting boolSetting = Setting.boolSetting("boolSetting", false, Setting.Property.ExtensionScope); + Setting stringSetting = Setting.simpleString("stringSetting", "default", Setting.Property.ExtensionScope); + Setting intSetting = Setting.intSetting("intSetting", 0, Setting.Property.ExtensionScope); + Setting listSetting = Setting.listSetting( + "listSetting", + List.of("first", "second", "third"), + Function.identity(), + Setting.Property.ExtensionScope + ); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager spy = spy(extensionsManager); + + // optionally, you can stub out some methods: + when(spy.getAdditionalSettings()).thenCallRealMethod(); + Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); + Mockito.doNothing().when(spy).initialize(); + RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); + final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" + + Version.CURRENT.toString() + + "\"," + + "\"minimumCompatibleVersion\":\"" + + Version.CURRENT.minimumCompatibilityVersion().toString() + + "\"}"; + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(content), XContentType.JSON) + .withMethod(RestRequest.Method.POST) + .build(); + + FakeRestChannel channel = new FakeRestChannel(request, false, 0); + restInitializeExtensionAction.handleRequest(request, channel, null); + + assertEquals(channel.capturedResponse().status(), RestStatus.ACCEPTED); + assertTrue(channel.capturedResponse().content().utf8ToString().contains("A request to initialize an extension has been sent.")); + + Optional extension = spy.lookupExtensionSettingsById("ad-extension"); + assertTrue(extension.isPresent()); + assertEquals(false, extension.get().getAdditionalSettings().get(boolSetting)); + assertEquals("default", extension.get().getAdditionalSettings().get(stringSetting)); + assertEquals(0, extension.get().getAdditionalSettings().get(intSetting)); + + List listSettingValue = (List) extension.get().getAdditionalSettings().get(listSetting); + assertTrue(listSettingValue.contains("first")); + assertTrue(listSettingValue.contains("second")); + assertTrue(listSettingValue.contains("third")); + } + } From 1f3a7a87eb64f8280bf984eed8fa821f1351b583 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 4 Jul 2023 12:19:59 -0400 Subject: [PATCH 044/186] Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies (#8434) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 8 ++++---- client/rest/licenses/httpclient5-5.1.4.jar.sha1 | 1 - client/rest/licenses/httpclient5-5.2.1.jar.sha1 | 1 + client/rest/licenses/httpcore5-5.1.5.jar.sha1 | 1 - client/rest/licenses/httpcore5-5.2.2.jar.sha1 | 1 + client/rest/licenses/httpcore5-h2-5.1.5.jar.sha1 | 1 - client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1 | 1 + client/sniffer/licenses/httpclient5-5.1.4.jar.sha1 | 1 - client/sniffer/licenses/httpclient5-5.2.1.jar.sha1 | 1 + client/sniffer/licenses/httpcore5-5.1.5.jar.sha1 | 1 - client/sniffer/licenses/httpcore5-5.2.2.jar.sha1 | 1 + .../licenses/httpclient-4.5.13.jar.sha1 | 1 - .../licenses/httpclient-4.5.14.jar.sha1 | 1 + .../licenses/httpcore-4.4.15.jar.sha1 | 1 - .../licenses/httpcore-4.4.16.jar.sha1 | 1 + plugins/discovery-ec2/licenses/httpclient-4.5.13.jar.sha1 | 1 - plugins/discovery-ec2/licenses/httpclient-4.5.14.jar.sha1 | 1 + plugins/discovery-ec2/licenses/httpcore-4.4.15.jar.sha1 | 1 - plugins/discovery-ec2/licenses/httpcore-4.4.16.jar.sha1 | 1 + plugins/discovery-gce/licenses/httpclient-4.5.13.jar.sha1 | 1 - plugins/discovery-gce/licenses/httpclient-4.5.14.jar.sha1 | 1 + plugins/discovery-gce/licenses/httpcore-4.4.15.jar.sha1 | 1 - plugins/discovery-gce/licenses/httpcore-4.4.16.jar.sha1 | 1 + plugins/repository-s3/licenses/httpclient-4.5.13.jar.sha1 | 1 - plugins/repository-s3/licenses/httpclient-4.5.14.jar.sha1 | 1 + plugins/repository-s3/licenses/httpcore-4.4.15.jar.sha1 | 1 - plugins/repository-s3/licenses/httpcore-4.4.16.jar.sha1 | 1 + 28 files changed, 18 insertions(+), 17 deletions(-) delete mode 100644 client/rest/licenses/httpclient5-5.1.4.jar.sha1 create mode 100644 client/rest/licenses/httpclient5-5.2.1.jar.sha1 delete mode 100644 client/rest/licenses/httpcore5-5.1.5.jar.sha1 create mode 100644 client/rest/licenses/httpcore5-5.2.2.jar.sha1 delete mode 100644 client/rest/licenses/httpcore5-h2-5.1.5.jar.sha1 create mode 100644 client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1 delete mode 100644 client/sniffer/licenses/httpclient5-5.1.4.jar.sha1 create mode 100644 client/sniffer/licenses/httpclient5-5.2.1.jar.sha1 delete mode 100644 client/sniffer/licenses/httpcore5-5.1.5.jar.sha1 create mode 100644 client/sniffer/licenses/httpcore5-5.2.2.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/httpclient-4.5.13.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/httpclient-4.5.14.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/httpcore-4.4.15.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/httpcore-4.4.16.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.13.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.14.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.15.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.16.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.13.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.14.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.15.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.16.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/httpclient-4.5.13.jar.sha1 create mode 100644 plugins/repository-s3/licenses/httpclient-4.5.14.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/httpcore-4.4.15.jar.sha1 create mode 100644 plugins/repository-s3/licenses/httpcore-4.4.16.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8deb28833e6..1d10c55fc8064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,6 +128,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Upgrade] Lucene 9.7.0 release (#8272) - Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209)) - Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276)) +- Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434)) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index f9eac9516cb18..408b03e60cc5d 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -33,10 +33,10 @@ netty = 4.1.94.Final joda = 2.12.2 # client dependencies -httpclient5 = 5.1.4 -httpcore5 = 5.1.5 -httpclient = 4.5.13 -httpcore = 4.4.15 +httpclient5 = 5.2.1 +httpcore5 = 5.2.2 +httpclient = 4.5.14 +httpcore = 4.4.16 httpasyncclient = 4.1.5 commonslogging = 1.2 commonscodec = 1.15 diff --git a/client/rest/licenses/httpclient5-5.1.4.jar.sha1 b/client/rest/licenses/httpclient5-5.1.4.jar.sha1 deleted file mode 100644 index 3c0cb1335fb88..0000000000000 --- a/client/rest/licenses/httpclient5-5.1.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -208f9eed6d6ab709e2ae7a75b457ef60c0baefa5 \ No newline at end of file diff --git a/client/rest/licenses/httpclient5-5.2.1.jar.sha1 b/client/rest/licenses/httpclient5-5.2.1.jar.sha1 new file mode 100644 index 0000000000000..3555fe22f8e12 --- /dev/null +++ b/client/rest/licenses/httpclient5-5.2.1.jar.sha1 @@ -0,0 +1 @@ +0c900514d3446d9ce5d9dbd90c21192048125440 \ No newline at end of file diff --git a/client/rest/licenses/httpcore5-5.1.5.jar.sha1 b/client/rest/licenses/httpcore5-5.1.5.jar.sha1 deleted file mode 100644 index 8da253152e970..0000000000000 --- a/client/rest/licenses/httpcore5-5.1.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df9da3a1fa2351c4790245400ed28d78a8ddd3fc \ No newline at end of file diff --git a/client/rest/licenses/httpcore5-5.2.2.jar.sha1 b/client/rest/licenses/httpcore5-5.2.2.jar.sha1 new file mode 100644 index 0000000000000..b641256c7d4a4 --- /dev/null +++ b/client/rest/licenses/httpcore5-5.2.2.jar.sha1 @@ -0,0 +1 @@ +6da28f5aa6c2b129ef49632e041a5203ce7507b2 \ No newline at end of file diff --git a/client/rest/licenses/httpcore5-h2-5.1.5.jar.sha1 b/client/rest/licenses/httpcore5-h2-5.1.5.jar.sha1 deleted file mode 100644 index 097e6cc2a3be8..0000000000000 --- a/client/rest/licenses/httpcore5-h2-5.1.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -624660339afd5006d427457e6b10b10b32fd86f1 \ No newline at end of file diff --git a/client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1 b/client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1 new file mode 100644 index 0000000000000..94bc0fa49bdb0 --- /dev/null +++ b/client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1 @@ -0,0 +1 @@ +54ee1ed58fe8ac40be1083ea9873a6c734939ab9 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient5-5.1.4.jar.sha1 b/client/sniffer/licenses/httpclient5-5.1.4.jar.sha1 deleted file mode 100644 index 3c0cb1335fb88..0000000000000 --- a/client/sniffer/licenses/httpclient5-5.1.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -208f9eed6d6ab709e2ae7a75b457ef60c0baefa5 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient5-5.2.1.jar.sha1 b/client/sniffer/licenses/httpclient5-5.2.1.jar.sha1 new file mode 100644 index 0000000000000..3555fe22f8e12 --- /dev/null +++ b/client/sniffer/licenses/httpclient5-5.2.1.jar.sha1 @@ -0,0 +1 @@ +0c900514d3446d9ce5d9dbd90c21192048125440 \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore5-5.1.5.jar.sha1 b/client/sniffer/licenses/httpcore5-5.1.5.jar.sha1 deleted file mode 100644 index 8da253152e970..0000000000000 --- a/client/sniffer/licenses/httpcore5-5.1.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df9da3a1fa2351c4790245400ed28d78a8ddd3fc \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore5-5.2.2.jar.sha1 b/client/sniffer/licenses/httpcore5-5.2.2.jar.sha1 new file mode 100644 index 0000000000000..b641256c7d4a4 --- /dev/null +++ b/client/sniffer/licenses/httpcore5-5.2.2.jar.sha1 @@ -0,0 +1 @@ +6da28f5aa6c2b129ef49632e041a5203ce7507b2 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpclient-4.5.13.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.13.jar.sha1 deleted file mode 100644 index 3281e21595b39..0000000000000 --- a/plugins/discovery-azure-classic/licenses/httpclient-4.5.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpclient-4.5.14.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.14.jar.sha1 new file mode 100644 index 0000000000000..66e05851c2e3c --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpclient-4.5.14.jar.sha1 @@ -0,0 +1 @@ +1194890e6f56ec29177673f2f12d0b8e627dec98 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.15.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.15.jar.sha1 deleted file mode 100644 index 42a03b5d7a376..0000000000000 --- a/plugins/discovery-azure-classic/licenses/httpcore-4.4.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f2e0c573eaa7a74bac2e89b359e1f73d92a0a1d \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.16.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.16.jar.sha1 new file mode 100644 index 0000000000000..172110694b5bd --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpcore-4.4.16.jar.sha1 @@ -0,0 +1 @@ +51cf043c87253c9f58b539c9f7e44c8894223850 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.13.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.13.jar.sha1 deleted file mode 100644 index 3281e21595b39..0000000000000 --- a/plugins/discovery-ec2/licenses/httpclient-4.5.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.14.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.14.jar.sha1 new file mode 100644 index 0000000000000..66e05851c2e3c --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpclient-4.5.14.jar.sha1 @@ -0,0 +1 @@ +1194890e6f56ec29177673f2f12d0b8e627dec98 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.15.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.15.jar.sha1 deleted file mode 100644 index 42a03b5d7a376..0000000000000 --- a/plugins/discovery-ec2/licenses/httpcore-4.4.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f2e0c573eaa7a74bac2e89b359e1f73d92a0a1d \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.16.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.16.jar.sha1 new file mode 100644 index 0000000000000..172110694b5bd --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpcore-4.4.16.jar.sha1 @@ -0,0 +1 @@ +51cf043c87253c9f58b539c9f7e44c8894223850 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.13.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.13.jar.sha1 deleted file mode 100644 index 3281e21595b39..0000000000000 --- a/plugins/discovery-gce/licenses/httpclient-4.5.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.14.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.14.jar.sha1 new file mode 100644 index 0000000000000..66e05851c2e3c --- /dev/null +++ b/plugins/discovery-gce/licenses/httpclient-4.5.14.jar.sha1 @@ -0,0 +1 @@ +1194890e6f56ec29177673f2f12d0b8e627dec98 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.15.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.15.jar.sha1 deleted file mode 100644 index 42a03b5d7a376..0000000000000 --- a/plugins/discovery-gce/licenses/httpcore-4.4.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f2e0c573eaa7a74bac2e89b359e1f73d92a0a1d \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.16.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.16.jar.sha1 new file mode 100644 index 0000000000000..172110694b5bd --- /dev/null +++ b/plugins/discovery-gce/licenses/httpcore-4.4.16.jar.sha1 @@ -0,0 +1 @@ +51cf043c87253c9f58b539c9f7e44c8894223850 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpclient-4.5.13.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.13.jar.sha1 deleted file mode 100644 index 3281e21595b39..0000000000000 --- a/plugins/repository-s3/licenses/httpclient-4.5.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpclient-4.5.14.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.14.jar.sha1 new file mode 100644 index 0000000000000..66e05851c2e3c --- /dev/null +++ b/plugins/repository-s3/licenses/httpclient-4.5.14.jar.sha1 @@ -0,0 +1 @@ +1194890e6f56ec29177673f2f12d0b8e627dec98 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.15.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.15.jar.sha1 deleted file mode 100644 index 42a03b5d7a376..0000000000000 --- a/plugins/repository-s3/licenses/httpcore-4.4.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f2e0c573eaa7a74bac2e89b359e1f73d92a0a1d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.16.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.16.jar.sha1 new file mode 100644 index 0000000000000..172110694b5bd --- /dev/null +++ b/plugins/repository-s3/licenses/httpcore-4.4.16.jar.sha1 @@ -0,0 +1 @@ +51cf043c87253c9f58b539c9f7e44c8894223850 \ No newline at end of file From 71c9302c022d7fb8d461b0ec7e7810e0e04d0799 Mon Sep 17 00:00:00 2001 From: Ashish Date: Wed, 5 Jul 2023 10:45:12 +0530 Subject: [PATCH 045/186] [Remote translog] Add integration tests for primary term validation (#8406) --------- Signed-off-by: Ashish Singh --- .../remotestore/PrimaryTermValidationIT.java | 166 ++++++++++++++++++ .../RemoteStoreBaseIntegTestCase.java | 4 +- .../opensearch/remotestore/RemoteStoreIT.java | 6 + .../remotestore/RemoteStoreStatsIT.java | 6 + .../ReplicaToPrimaryPromotionIT.java | 6 + 5 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java new file mode 100644 index 0000000000000..6691da81f057d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -0,0 +1,166 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.cluster.coordination.FollowersChecker; +import org.opensearch.cluster.coordination.LeaderChecker; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.health.ClusterIndexHealth; +import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.shard.ShardNotFoundException; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.disruption.NetworkDisruption; +import org.opensearch.test.transport.MockTransportService; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) + +public class PrimaryTermValidationIT extends RemoteStoreBaseIntegTestCase { + + private static final String INDEX_NAME = "remote-store-test-idx-1"; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockTransportService.TestPlugin.class); + } + + public void testPrimaryTermValidation() throws Exception { + // Follower checker interval is lower compared to leader checker so that the cluster manager can remove the node + // with network partition faster. The follower check retry count is also kept 1. + Settings clusterSettings = Settings.builder() + .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") + .put(LeaderChecker.LEADER_CHECK_INTERVAL_SETTING.getKey(), "20s") + .put(LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), 4) + .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "1s") + .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "1s") + .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) + .build(); + internalCluster().startClusterManagerOnlyNode(clusterSettings); + + // Create repository + absolutePath = randomRepoPath().toAbsolutePath(); + assertAcked( + clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) + ); + + // Start data nodes and create index + internalCluster().startDataOnlyNodes(2, clusterSettings); + createIndex(INDEX_NAME, remoteTranslogIndexSettings(1)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + // Get the names of nodes to create network disruption + String primaryNode = primaryNodeName(INDEX_NAME); + String replicaNode = replicaNodeName(INDEX_NAME); + String clusterManagerNode = internalCluster().getClusterManagerName(); + logger.info("Node names : clusterManager={} primary={} replica={}", clusterManagerNode, primaryNode, replicaNode); + + // Index some docs and validate that both primary and replica node has it. Refresh is triggered to trigger segment replication + // to ensure replica is also upto date. + int numOfDocs = randomIntBetween(5, 10); + for (int i = 0; i < numOfDocs; i++) { + indexSameDoc(clusterManagerNode, INDEX_NAME); + } + refresh(INDEX_NAME); + assertBusy( + () -> assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), numOfDocs) + ); + assertBusy( + () -> assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), numOfDocs) + ); + + // Start network disruption - primary node will be isolated + Set nodesInOneSide = Stream.of(clusterManagerNode, replicaNode).collect(Collectors.toCollection(HashSet::new)); + Set nodesInOtherSide = Stream.of(primaryNode).collect(Collectors.toCollection(HashSet::new)); + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide), + NetworkDisruption.DISCONNECT + ); + internalCluster().setDisruptionScheme(networkDisruption); + logger.info("--> network disruption is started"); + networkDisruption.startDisrupting(); + + // Ensure the node which is partitioned is removed from the cluster + assertBusy(() -> { + NodesInfoResponse response = client(clusterManagerNode).admin().cluster().prepareNodesInfo().get(); + assertThat(response.getNodes().size(), equalTo(2)); + }); + + // Ensure that the cluster manager has latest information about the index + assertBusy(() -> { + ClusterHealthResponse clusterHealthResponse = client(clusterManagerNode).admin() + .cluster() + .health(new ClusterHealthRequest()) + .actionGet(TimeValue.timeValueSeconds(1)); + assertTrue(clusterHealthResponse.getIndices().containsKey(INDEX_NAME)); + ClusterIndexHealth clusterIndexHealth = clusterHealthResponse.getIndices().get(INDEX_NAME); + assertEquals(ClusterHealthStatus.YELLOW, clusterHealthResponse.getStatus()); + assertEquals(1, clusterIndexHealth.getNumberOfShards()); + assertEquals(1, clusterIndexHealth.getActiveShards()); + assertEquals(1, clusterIndexHealth.getUnassignedShards()); + assertEquals(1, clusterIndexHealth.getUnassignedShards()); + assertEquals(1, clusterIndexHealth.getActivePrimaryShards()); + assertEquals(ClusterHealthStatus.YELLOW, clusterIndexHealth.getStatus()); + }); + + // Index data to the newly promoted primary + indexSameDoc(clusterManagerNode, INDEX_NAME); + RefreshResponse refreshResponse = client(clusterManagerNode).admin() + .indices() + .prepareRefresh(INDEX_NAME) + .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED) + .execute() + .actionGet(); + assertNoFailures(refreshResponse); + assertEquals(1, refreshResponse.getSuccessfulShards()); + assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), numOfDocs + 1); + + // At this point we stop the disruption. Since the follower checker has already failed and cluster manager has removed the node + // from cluster, failed node needs to start discovery process by leader checker call. We stop the disruption to allow the failed + // node to + // communicate with the other node which it assumes has replica. + networkDisruption.stopDisrupting(); + + // When the index call is made to the stale primary, it makes the primary term validation call to the other node (which + // it assumes has the replica node). At this moment, the stale primary realises that it is no more the primary and the caller + // received the following exception. + ShardNotFoundException exception = assertThrows(ShardNotFoundException.class, () -> indexSameDoc(primaryNode, INDEX_NAME)); + assertTrue(exception.getMessage().contains("no such shard")); + ensureStableCluster(3); + ensureGreen(INDEX_NAME); + } + + private IndexResponse indexSameDoc(String nodeName, String indexName) { + return client(nodeName).prepareIndex(indexName) + .setId(UUIDs.randomBase64UUID()) + .setSource("{\"foo\" : \"bar\"}", XContentType.JSON) + .get(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 336646b35b5a6..2b3fcadfc645e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -9,7 +9,6 @@ package org.opensearch.remotestore; import org.junit.After; -import org.junit.Before; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -100,8 +99,7 @@ protected void putRepository(Path path) { ); } - @Before - public void setup() { + protected void setupRepo() { internalCluster().startClusterManagerOnlyNode(); absolutePath = randomRepoPath().toAbsolutePath(); assertAcked( diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index f6ba8cfed00d0..77de601b53ec6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.junit.Before; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; @@ -51,6 +52,11 @@ protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } + @Before + public void setup() { + setupRepo(); + } + @Override public Settings indexSettings() { return remoteStoreIndexSettings(0); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 0ea87d106c14e..0e4774c1f3454 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.junit.Before; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsRequestBuilder; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; @@ -28,6 +29,11 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; + @Before + public void setup() { + setupRepo(); + } + public void testStatsResponseFromAllNodes() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index 712747f7479ae..0f3e041dd429a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -9,6 +9,7 @@ package org.opensearch.remotestore; import com.carrotsearch.randomizedtesting.RandomizedTest; +import org.junit.Before; import org.opensearch.action.admin.indices.close.CloseIndexResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -31,6 +32,11 @@ public class ReplicaToPrimaryPromotionIT extends RemoteStoreBaseIntegTestCase { private int shard_count = 5; + @Before + public void setup() { + setupRepo(); + } + @Override public Settings indexSettings() { return Settings.builder() From 5fd49d0f3052ccc37553c236e8016ce7a6a344a7 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Wed, 5 Jul 2023 12:05:15 +0530 Subject: [PATCH 046/186] Adds mock implementation for TelemetryPlugin (#8357) --------- Signed-off-by: Gagan Juneja Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 1 + .../main/java/org/opensearch/node/Node.java | 2 +- settings.gradle | 3 +- test/framework/build.gradle | 1 + .../test/OpenSearchIntegTestCase.java | 4 + .../test/OpenSearchSingleNodeTestCase.java | 6 + .../test/telemetry/MockTelemetry.java | 42 +++++ .../test/telemetry/MockTelemetryPlugin.java | 39 +++++ test/telemetry/build.gradle | 25 +++ .../test/telemetry/tracing/MockSpan.java | 159 ++++++++++++++++++ .../tracing/MockTracingContextPropagator.java | 56 ++++++ .../tracing/MockTracingTelemetry.java | 46 +++++ .../test/telemetry/tracing/SpanProcessor.java | 28 +++ .../tracing/StrictCheckSpanProcessor.java | 84 +++++++++ .../test/telemetry/tracing/package-info.java | 10 ++ 15 files changed, 504 insertions(+), 2 deletions(-) create mode 100644 test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java create mode 100644 test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetryPlugin.java create mode 100644 test/telemetry/build.gradle create mode 100644 test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java create mode 100644 test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java create mode 100644 test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java create mode 100644 test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/SpanProcessor.java create mode 100644 test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java create mode 100644 test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/package-info.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d10c55fc8064..24c86bf6ff714 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) - Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) - [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) +- Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index d3655671b516d..aecf5659de7fe 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1501,7 +1501,7 @@ public synchronized void close() throws IOException { toClose.add(injector.getInstance(NodeEnvironment.class)); toClose.add(stopWatch::stop); if (FeatureFlags.isEnabled(TELEMETRY)) { - toClose.add(() -> injector.getInstance(TracerFactory.class)); + toClose.add(injector.getInstance(TracerFactory.class)); } if (logger.isTraceEnabled()) { diff --git a/settings.gradle b/settings.gradle index bf899d04c1e08..d908e941b3d6b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -87,7 +87,8 @@ List projects = [ 'test:fixtures:minio-fixture', 'test:fixtures:old-elasticsearch', 'test:fixtures:s3-fixture', - 'test:logger-usage' + 'test:logger-usage', + 'test:telemetry' ] /** diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 255f554db7a79..2532fdf1938fd 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -38,6 +38,7 @@ dependencies { api project(':libs:opensearch-nio') api project(":server") api project(":libs:opensearch-cli") + api project(":test:telemetry") api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" api "junit:junit:${versions.junit}" api "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index caa5b90016740..fec45219ace81 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -109,6 +109,7 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -152,6 +153,7 @@ import org.opensearch.test.disruption.ServiceDisruptionScheme; import org.opensearch.test.store.MockFSIndexStore; import org.opensearch.test.transport.MockTransportService; +import org.opensearch.test.telemetry.MockTelemetryPlugin; import org.opensearch.transport.TransportInterceptor; import org.opensearch.transport.TransportRequest; import org.opensearch.transport.TransportRequestHandler; @@ -776,6 +778,7 @@ protected Settings featureFlagSettings() { for (Setting builtInFlag : FeatureFlagSettings.BUILT_IN_FEATURE_FLAGS) { featureSettings.put(builtInFlag.getKey(), builtInFlag.getDefaultRaw(Settings.EMPTY)); } + featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true); return featureSettings.build(); } @@ -2101,6 +2104,7 @@ protected Collection> getMockPlugins() { if (addMockGeoShapeFieldMapper()) { mocks.add(TestGeoShapeFieldMapperPlugin.class); } + mocks.add(MockTelemetryPlugin.class); return Collections.unmodifiableList(mocks); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java index bf797ef6b310b..91c48f1679f9a 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java @@ -48,6 +48,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.Strings; @@ -66,6 +67,8 @@ import org.opensearch.plugins.Plugin; import org.opensearch.script.MockScriptService; import org.opensearch.search.internal.SearchContext; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.test.telemetry.MockTelemetryPlugin; import org.opensearch.transport.TransportSettings; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -242,6 +245,8 @@ private Node newNode() { .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false) .putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()) // empty list disables a port scan for other nodes .putList(INITIAL_CLUSTER_MANAGER_NODES_SETTING.getKey(), nodeName) + .put(FeatureFlags.TELEMETRY_SETTING.getKey(), true) + .put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true) .put(nodeSettings()) // allow test cases to provide their own settings or override these .build(); @@ -254,6 +259,7 @@ private Node newNode() { plugins.add(MockHttpTransport.TestPlugin.class); } plugins.add(MockScriptService.TestPlugin.class); + plugins.add(MockTelemetryPlugin.class); Node node = new MockNode(settings, plugins, forbidPrivateIndexSettings()); try { node.start(); diff --git a/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java new file mode 100644 index 0000000000000..c02ab1d737303 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry; + +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.metrics.MetricsTelemetry; +import org.opensearch.test.telemetry.tracing.MockTracingTelemetry; +import org.opensearch.telemetry.tracing.TracingTelemetry; + +/** + * Mock {@link Telemetry} implementation for testing. + */ +public class MockTelemetry implements Telemetry { + + private final TelemetrySettings settings; + + /** + * Constructor with settings. + * @param settings telemetry settings. + */ + public MockTelemetry(TelemetrySettings settings) { + this.settings = settings; + } + + @Override + public TracingTelemetry getTracingTelemetry() { + return new MockTracingTelemetry(); + } + + @Override + public MetricsTelemetry getMetricsTelemetry() { + return new MetricsTelemetry() { + }; + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetryPlugin.java b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetryPlugin.java new file mode 100644 index 0000000000000..41cc5c1e77a34 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetryPlugin.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry; + +import java.util.Optional; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.TelemetryPlugin; +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; + +/** + * Mock {@link TelemetryPlugin} implementation for testing. + */ +public class MockTelemetryPlugin extends Plugin implements TelemetryPlugin { + private static final String MOCK_TRACER_NAME = "mock"; + + /** + * Base constructor. + */ + public MockTelemetryPlugin() { + + } + + @Override + public Optional getTelemetry(TelemetrySettings settings) { + return Optional.of(new MockTelemetry(settings)); + } + + @Override + public String getName() { + return MOCK_TRACER_NAME; + } +} diff --git a/test/telemetry/build.gradle b/test/telemetry/build.gradle new file mode 100644 index 0000000000000..fbabe43aa5e5a --- /dev/null +++ b/test/telemetry/build.gradle @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.build' +apply plugin: 'opensearch.publish' + +dependencies { + api project(":libs:opensearch-common") + api project(":libs:opensearch-telemetry") +} + +tasks.named('forbiddenApisMain').configure { + //package does not depend on core, so only jdk signatures should be checked + replaceSignatureFiles 'jdk-signatures' +} + +test.enabled = false diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java new file mode 100644 index 0000000000000..4779d9796e11e --- /dev/null +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java @@ -0,0 +1,159 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry.tracing; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Supplier; +import org.opensearch.telemetry.tracing.AbstractSpan; +import org.opensearch.telemetry.tracing.Span; + +/** + * MockSpan for testing and strict check validations. Not to be used for production cases. + */ +public class MockSpan extends AbstractSpan { + private final SpanProcessor spanProcessor; + private final Map metadata; + private final String traceId; + private final String spanId; + private boolean hasEnded; + private final Long startTime; + private Long endTime; + + private final Object lock = new Object(); + + private static final Supplier randomSupplier = ThreadLocalRandom::current; + + /** + * Base Constructor. + * @param spanName span name + * @param parentSpan parent span + * @param spanProcessor span processor + */ + public MockSpan(String spanName, Span parentSpan, SpanProcessor spanProcessor) { + this( + spanName, + parentSpan, + parentSpan != null ? parentSpan.getTraceId() : IdGenerator.generateTraceId(), + IdGenerator.generateSpanId(), + spanProcessor + ); + } + + /** + * Constructor with traceId and SpanIds + * @param spanName Span Name + * @param parentSpan Parent Span + * @param traceId Trace ID + * @param spanId Span ID + * @param spanProcessor Span Processor + */ + public MockSpan(String spanName, Span parentSpan, String traceId, String spanId, SpanProcessor spanProcessor) { + super(spanName, parentSpan); + this.spanProcessor = spanProcessor; + this.metadata = new HashMap<>(); + this.traceId = traceId; + this.spanId = spanId; + this.startTime = System.nanoTime(); + } + + @Override + public void endSpan() { + synchronized (lock) { + if (hasEnded) { + return; + } + endTime = System.nanoTime(); + hasEnded = true; + } + spanProcessor.onEnd(this); + } + + @Override + public void addAttribute(String key, String value) { + putMetadata(key, value); + } + + @Override + public void addAttribute(String key, Long value) { + putMetadata(key, value); + } + + @Override + public void addAttribute(String key, Double value) { + putMetadata(key, value); + } + + @Override + public void addAttribute(String key, Boolean value) { + putMetadata(key, value); + } + + @Override + public void addEvent(String event) { + putMetadata(event, null); + } + + private void putMetadata(String key, Object value) { + metadata.put(key, value); + } + + @Override + public String getTraceId() { + return traceId; + } + + @Override + public String getSpanId() { + return spanId; + } + + /** + * Returns whether the span is ended or not. + * @return span end status. + */ + public boolean hasEnded() { + synchronized (lock) { + return hasEnded; + } + } + + /** + * Returns the start time of the span. + * @return start time of the span. + */ + public Long getStartTime() { + return startTime; + } + + /** + * Returns the start time of the span. + * @return end time of the span. + */ + public Long getEndTime() { + return endTime; + } + + private static class IdGenerator { + private static String generateSpanId() { + long id = randomSupplier.get().nextLong(); + return Long.toHexString(id); + } + + private static String generateTraceId() { + long idHi = randomSupplier.get().nextLong(); + long idLo = randomSupplier.get().nextLong(); + long result = idLo | (idHi << 32); + return Long.toHexString(result); + } + + } +} diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java new file mode 100644 index 0000000000000..7e3f5a9031100 --- /dev/null +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry.tracing; + +import java.util.Locale; +import java.util.Map; +import java.util.function.BiConsumer; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.TracingContextPropagator; + +/** + * Mock {@link TracingContextPropagator} to persist the span for internode communication. + */ +public class MockTracingContextPropagator implements TracingContextPropagator { + + private static final String TRACE_PARENT = "traceparent"; + private static final String SEPARATOR = "~"; + private final SpanProcessor spanProcessor; + + /** + * Constructor + * @param spanProcessor span processor. + */ + public MockTracingContextPropagator(SpanProcessor spanProcessor) { + this.spanProcessor = spanProcessor; + } + + @Override + public Span extract(Map props) { + String value = props.get(TRACE_PARENT); + if (value != null) { + String[] values = value.split(SEPARATOR); + String traceId = values[0]; + String spanId = values[1]; + return new MockSpan(null, null, traceId, spanId, spanProcessor); + } else { + return null; + } + } + + @Override + public void inject(Span currentSpan, BiConsumer setter) { + if (currentSpan instanceof MockSpan) { + String traceId = currentSpan.getTraceId(); + String spanId = currentSpan.getSpanId(); + String traceParent = String.format(Locale.ROOT, "%s%s%s", traceId, SEPARATOR, spanId); + setter.accept(TRACE_PARENT, traceParent); + } + } +} diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java new file mode 100644 index 0000000000000..531b4ce36c36a --- /dev/null +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry.tracing; + +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.TracingContextPropagator; +import org.opensearch.telemetry.tracing.TracingTelemetry; + +/** + * Mock {@link TracingTelemetry} implementation for testing. + */ +public class MockTracingTelemetry implements TracingTelemetry { + + private final SpanProcessor spanProcessor = new StrictCheckSpanProcessor(); + + /** + * Base constructor. + */ + public MockTracingTelemetry() { + + } + + @Override + public Span createSpan(String spanName, Span parentSpan) { + Span span = new MockSpan(spanName, parentSpan, spanProcessor); + spanProcessor.onStart(span); + return span; + } + + @Override + public TracingContextPropagator getContextPropagator() { + return new MockTracingContextPropagator(spanProcessor); + } + + @Override + public void close() { + ((StrictCheckSpanProcessor) spanProcessor).ensureAllSpansAreClosed(); + ((StrictCheckSpanProcessor) spanProcessor).clear(); + } +} diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/SpanProcessor.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/SpanProcessor.java new file mode 100644 index 0000000000000..cb9d0dbd428e4 --- /dev/null +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/SpanProcessor.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry.tracing; + +import org.opensearch.telemetry.tracing.Span; + +/** + * Processes the span and can perform any action on the span start and end. + */ +public interface SpanProcessor { + /** + * Logic to be executed on span start. + * @param span span which is starting. + */ + void onStart(Span span); + + /** + * Logic to be executed on span end. + * @param span span which is ending. + */ + void onEnd(Span span); +} diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java new file mode 100644 index 0000000000000..34d4d96809755 --- /dev/null +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test.telemetry.tracing; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.opensearch.telemetry.tracing.Span; + +/** + * Strict check span processor to validate the spans. + */ +public class StrictCheckSpanProcessor implements SpanProcessor { + private final Map spanMap = new ConcurrentHashMap<>(); + + /** + * Base constructor. + */ + public StrictCheckSpanProcessor() { + + } + + @Override + public void onStart(Span span) { + spanMap.put(span.getSpanId(), Thread.currentThread().getStackTrace()); + } + + @Override + public void onEnd(Span span) { + spanMap.remove(span.getSpanId()); + } + + /** + * Ensures that all the spans are closed. Throws exception message with stack trace of the method form + * where the span was created. We can enhance it to print all the failed spans in a single go based on + * the usability. + */ + public void ensureAllSpansAreClosed() { + if (!spanMap.isEmpty()) { + for (Map.Entry entry : spanMap.entrySet()) { + StackTraceElement[] filteredStackTrace = getFilteredStackTrace(entry.getValue()); + AssertionError error = new AssertionError( + String.format( + Locale.ROOT, + " Total [%d] spans are not ended properly. " + "Find below the stack trace for one of the un-ended span", + spanMap.size() + ) + ); + error.setStackTrace(filteredStackTrace); + spanMap.clear(); + throw error; + } + } + } + + /** + * Clears the state. + */ + public void clear() { + spanMap.clear(); + } + + private StackTraceElement[] getFilteredStackTrace(StackTraceElement[] stackTraceElements) { + int filteredElementsCount = 0; + while (filteredElementsCount < stackTraceElements.length) { + String className = stackTraceElements[filteredElementsCount].getClassName(); + if (className.startsWith("java.lang.Thread") + || className.startsWith("org.opensearch.telemetry") + || className.startsWith("org.opensearch.tracing")) { + filteredElementsCount++; + } else { + break; + } + } + return Arrays.copyOfRange(stackTraceElements, filteredElementsCount, stackTraceElements.length); + } +} diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/package-info.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/package-info.java new file mode 100644 index 0000000000000..83e2b4035acbf --- /dev/null +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Base opensearch package. */ +package org.opensearch.test.telemetry.tracing; From 245f9da562fef3ffe3d0071998f3fcc8a04e4403 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 00:57:28 -0700 Subject: [PATCH 047/186] Bump org.apache.maven:maven-model from 3.9.2 to 3.9.3 (#8403) * Bump org.apache.maven:maven-model from 3.9.2 to 3.9.3 Bumps [org.apache.maven:maven-model](https://github.com/apache/maven) from 3.9.2 to 3.9.3. - [Release notes](https://github.com/apache/maven/releases) - [Commits](https://github.com/apache/maven/compare/maven-3.9.2...maven-3.9.3) --- updated-dependencies: - dependency-name: org.apache.maven:maven-model dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 3 ++- buildSrc/build.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24c86bf6ff714..f122b260db91a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209)) - Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276)) - Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434)) +- Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) @@ -166,4 +167,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 35f3fb87560e7..47d4747d4b9a6 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -117,7 +117,7 @@ dependencies { api 'de.thetaphi:forbiddenapis:3.5.1' api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" - api 'org.apache.maven:maven-model:3.9.2' + api 'org.apache.maven:maven-model:3.9.3' api 'com.networknt:json-schema-validator:1.0.85' api 'org.jruby.jcodings:jcodings:1.0.58' api 'org.jruby.joni:joni:2.2.1' From c9974a4e202b56689954067881092a77e4aec83d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 01:35:34 -0700 Subject: [PATCH 048/186] Bump org.apache.avro:avro from 1.11.1 to 1.11.2 in /plugins/repository-hdfs (#8404) * Bump org.apache.avro:avro in /plugins/repository-hdfs Bumps org.apache.avro:avro from 1.11.1 to 1.11.2. --- updated-dependencies: - dependency-name: org.apache.avro:avro dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update Changelog entry. Signed-off-by: Rishikesh1159 * Update Changelog entry. Signed-off-by: Rishikesh1159 --------- Signed-off-by: dependabot[bot] Signed-off-by: Rishikesh1159 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Rishikesh1159 --- CHANGELOG.md | 2 +- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/avro-1.11.1.jar.sha1 | 1 - plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/avro-1.11.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index f122b260db91a..2a95531887c65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 - Bump `forbiddenapis` from 3.3 to 3.4 -- Bump `avro` from 1.11.0 to 1.11.1 +- Bump `avro` from 1.11.1 to 1.11.2 - Bump `woodstox-core` from 6.3.0 to 6.3.1 - Bump `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354)) - Bump `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447)) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 7ac54544f7a1b..3c83e535a91d8 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -66,7 +66,7 @@ dependencies { } api 'org.apache.htrace:htrace-core4:4.2.0-incubating' api "org.apache.logging.log4j:log4j-core:${versions.log4j}" - api 'org.apache.avro:avro:1.11.1' + api 'org.apache.avro:avro:1.11.2' api 'com.google.code.gson:gson:2.10.1' runtimeOnly "com.google.guava:guava:${versions.guava}" api "commons-logging:commons-logging:${versions.commonslogging}" diff --git a/plugins/repository-hdfs/licenses/avro-1.11.1.jar.sha1 b/plugins/repository-hdfs/licenses/avro-1.11.1.jar.sha1 deleted file mode 100644 index f03424516b44e..0000000000000 --- a/plugins/repository-hdfs/licenses/avro-1.11.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af5d4b9bdaaf4ba41bcb0df5241355ec34c630 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 b/plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 new file mode 100644 index 0000000000000..ce1a894e0ce6d --- /dev/null +++ b/plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 @@ -0,0 +1 @@ +97e62e8be2b37e849f1bdb5a4f08121d47cc9806 \ No newline at end of file From 39f3c35407d351a0f2ff5e79bb4f6326cc4941f5 Mon Sep 17 00:00:00 2001 From: suraj kumar Date: Wed, 5 Jul 2023 20:00:58 +0530 Subject: [PATCH 049/186] Move span operation to Scope from Tracer (#8411) * Move span operation to Scope from Tracer Signed-off-by: suranjay * Add changelog entry Signed-off-by: suranjay * Change method to setError Signed-off-by: suranjay * Removed unused classes Signed-off-by: suranjay * Fix merge issue Signed-off-by: suranjay --------- Signed-off-by: suranjay --- CHANGELOG.md | 1 + .../telemetry/tracing/DefaultSpanScope.java | 70 ++++++++++++++++ .../telemetry/tracing/DefaultTracer.java | 38 +-------- .../opensearch/telemetry/tracing/Scope.java | 26 ------ .../telemetry/tracing/ScopeImpl.java | 33 -------- .../opensearch/telemetry/tracing/Span.java | 7 ++ .../telemetry/tracing/SpanScope.java | 74 +++++++++++++++++ .../opensearch/telemetry/tracing/Tracer.java | 42 +--------- .../telemetry/tracing/noop/NoopSpanScope.java | 57 +++++++++++++ .../telemetry/tracing/noop/NoopTracer.java | 47 +---------- .../tracing/DefaultSpanScopeTests.java | 79 +++++++++++++++++++ .../telemetry/tracing/DefaultTracerTests.java | 47 +---------- .../telemetry/tracing/OTelSpan.java | 6 ++ .../telemetry/tracing/TracerFactoryTests.java | 2 +- .../test/telemetry/tracing/MockSpan.java | 4 + 15 files changed, 309 insertions(+), 224 deletions(-) create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java delete mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java delete mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanScope.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java create mode 100644 libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultSpanScopeTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a95531887c65..bb571cc21cd5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919)) - Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909)) +- Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) ### Deprecated diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java new file mode 100644 index 0000000000000..58e9e0abad739 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import java.util.function.Consumer; + +/** + * Default implementation of Scope + */ +public class DefaultSpanScope implements SpanScope { + + private final Span span; + + private final Consumer onCloseConsumer; + + /** + * Creates Scope instance for the given span + * + * @param span underlying span + * @param onCloseConsumer consumer to execute on scope close + */ + public DefaultSpanScope(Span span, Consumer onCloseConsumer) { + this.span = span; + this.onCloseConsumer = onCloseConsumer; + } + + @Override + public void addSpanAttribute(String key, String value) { + span.addAttribute(key, value); + } + + @Override + public void addSpanAttribute(String key, long value) { + span.addAttribute(key, value); + } + + @Override + public void addSpanAttribute(String key, double value) { + span.addAttribute(key, value); + } + + @Override + public void addSpanAttribute(String key, boolean value) { + span.addAttribute(key, value); + } + + @Override + public void addSpanEvent(String event) { + span.addEvent(event); + } + + @Override + public void setError(Exception exception) { + span.setError(exception); + } + + /** + * Executes the runnable to end the scope + */ + @Override + public void close() { + onCloseConsumer.accept(span); + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java index ab9110af7c3ab..783edd238c1c2 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java @@ -13,8 +13,8 @@ /** * - * The default tracer implementation. This class implements the basic logic for span lifecycle and its state management. - * It also handles tracing context propagation between spans. + * The default tracer implementation. It handles tracing context propagation between spans by maintaining + * current active span in its storage * * */ @@ -36,41 +36,11 @@ public DefaultTracer(TracingTelemetry tracingTelemetry, TracerContextStorage endSpan(span)); - } - - @Override - public void addSpanAttribute(String key, String value) { - Span currentSpan = getCurrentSpan(); - currentSpan.addAttribute(key, value); - } - - @Override - public void addSpanAttribute(String key, long value) { - Span currentSpan = getCurrentSpan(); - currentSpan.addAttribute(key, value); - } - - @Override - public void addSpanAttribute(String key, double value) { - Span currentSpan = getCurrentSpan(); - currentSpan.addAttribute(key, value); - } - - @Override - public void addSpanAttribute(String key, boolean value) { - Span currentSpan = getCurrentSpan(); - currentSpan.addAttribute(key, value); - } - - @Override - public void addSpanEvent(String event) { - Span currentSpan = getCurrentSpan(); - currentSpan.addEvent(event); + return new DefaultSpanScope(span, (scopeSpan) -> endSpan(scopeSpan)); } @Override diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java deleted file mode 100644 index 52f4eaf648eea..0000000000000 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.telemetry.tracing; - -/** - * An auto-closeable that represents scope of the span. - * It is recommended that you use this class with a try-with-resources block: - */ -public interface Scope extends AutoCloseable { - /** - * No-op Scope implementation - */ - Scope NO_OP = () -> {}; - - /** - * closes the scope - */ - @Override - void close(); -} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java deleted file mode 100644 index 30a7ac7fa90e7..0000000000000 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.telemetry.tracing; - -/** - * Executes the runnable on close - */ -public class ScopeImpl implements Scope { - - private Runnable runnableOnClose; - - /** - * Creates Scope instance - * @param runnableOnClose runnable to execute on scope close - */ - public ScopeImpl(Runnable runnableOnClose) { - this.runnableOnClose = runnableOnClose; - } - - /** - * Executes the runnable to end the scope - */ - @Override - public void close() { - runnableOnClose.run(); - } -} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java index 0710b8a22a37f..d60b4e60adece 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java @@ -62,6 +62,13 @@ public interface Span { */ void addAttribute(String key, Boolean value); + /** + * Records error in the span + * + * @param exception exception to be recorded + */ + void setError(Exception exception); + /** * Adds an event in the span * diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanScope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanScope.java new file mode 100644 index 0000000000000..cf67165d889bc --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanScope.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.telemetry.tracing.noop.NoopSpanScope; + +/** + * An auto-closeable that represents scope of the span. + * It provides interface for all the span operations. + */ +public interface SpanScope extends AutoCloseable { + /** + * No-op Scope implementation + */ + SpanScope NO_OP = new NoopSpanScope(); + + /** + * Adds string attribute to the {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, String value); + + /** + * Adds long attribute to the {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, long value); + + /** + * Adds double attribute to the {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, double value); + + /** + * Adds boolean attribute to the {@link Span}. + * + * @param key attribute key + * @param value attribute value + */ + void addSpanAttribute(String key, boolean value); + + /** + * Adds an event to the {@link Span}. + * + * @param event event name + */ + void addSpanEvent(String event); + + /** + * Records error in the span + * + * @param exception exception to be recorded + */ + void setError(Exception exception); + + /** + * closes the scope + */ + @Override + void close(); +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java index fcc091eb39c48..d422b58aa0a9f 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java @@ -11,7 +11,7 @@ import java.io.Closeable; /** - * Tracer is the interface used to create a {@link Span} and interact with current active {@link Span}. + * Tracer is the interface used to create a {@link Span} * It automatically handles the context propagation between threads, tasks, nodes etc. * * All methods on the Tracer object are multi-thread safe. @@ -24,44 +24,6 @@ public interface Tracer extends Closeable { * @param spanName span name * @return scope of the span, must be closed with explicit close or with try-with-resource */ - Scope startSpan(String spanName); + SpanScope startSpan(String spanName); - /** - * Adds string attribute to the current active {@link Span}. - * - * @param key attribute key - * @param value attribute value - */ - void addSpanAttribute(String key, String value); - - /** - * Adds long attribute to the current active {@link Span}. - * - * @param key attribute key - * @param value attribute value - */ - void addSpanAttribute(String key, long value); - - /** - * Adds double attribute to the current active {@link Span}. - * - * @param key attribute key - * @param value attribute value - */ - void addSpanAttribute(String key, double value); - - /** - * Adds boolean attribute to the current active {@link Span}. - * - * @param key attribute key - * @param value attribute value - */ - void addSpanAttribute(String key, boolean value); - - /** - * Adds an event to the current active {@link Span}. - * - * @param event event name - */ - void addSpanEvent(String event); } diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java new file mode 100644 index 0000000000000..c0dbaf65ba48b --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.noop; + +import org.opensearch.telemetry.tracing.SpanScope; + +/** + * No-op implementation of SpanScope + */ +public final class NoopSpanScope implements SpanScope { + + /** + * No-args constructor + */ + public NoopSpanScope() {} + + @Override + public void addSpanAttribute(String key, String value) { + + } + + @Override + public void addSpanAttribute(String key, long value) { + + } + + @Override + public void addSpanAttribute(String key, double value) { + + } + + @Override + public void addSpanAttribute(String key, boolean value) { + + } + + @Override + public void addSpanEvent(String event) { + + } + + @Override + public void setError(Exception exception) { + + } + + @Override + public void close() { + + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java index 18fc60e41e54d..a66cbcf4fef52 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java @@ -8,7 +8,7 @@ package org.opensearch.telemetry.tracing.noop; -import org.opensearch.telemetry.tracing.Scope; +import org.opensearch.telemetry.tracing.SpanScope; import org.opensearch.telemetry.tracing.Tracer; /** @@ -24,49 +24,8 @@ public class NoopTracer implements Tracer { private NoopTracer() {} @Override - public Scope startSpan(String spanName) { - return Scope.NO_OP; - } - - /** - * @param key attribute key - * @param value attribute value - */ - @Override - public void addSpanAttribute(String key, String value) { - - } - - /** - * @param key attribute key - * @param value attribute value - */ - @Override - public void addSpanAttribute(String key, long value) { - - } - - /** - * @param key attribute key - * @param value attribute value - */ - @Override - public void addSpanAttribute(String key, double value) { - - } - - /** - * @param key attribute key - * @param value attribute value - */ - @Override - public void addSpanAttribute(String key, boolean value) { - - } - - @Override - public void addSpanEvent(String event) { - + public SpanScope startSpan(String spanName) { + return SpanScope.NO_OP; } @Override diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultSpanScopeTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultSpanScopeTests.java new file mode 100644 index 0000000000000..eea6b77ce6e1e --- /dev/null +++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultSpanScopeTests.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.function.Consumer; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +public class DefaultSpanScopeTests extends OpenSearchTestCase { + + @SuppressWarnings("unchecked") + public void testClose() { + Span mockSpan = mock(Span.class); + Consumer mockConsumer = mock(Consumer.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, mockConsumer); + defaultSpanScope.close(); + + verify(mockConsumer).accept(mockSpan); + } + + public void testAddSpanAttributeString() { + Span mockSpan = mock(Span.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, null); + defaultSpanScope.addSpanAttribute("key", "value"); + + verify(mockSpan).addAttribute("key", "value"); + } + + public void testAddSpanAttributeLong() { + Span mockSpan = mock(Span.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, null); + defaultSpanScope.addSpanAttribute("key", 1L); + + verify(mockSpan).addAttribute("key", 1L); + } + + public void testAddSpanAttributeDouble() { + Span mockSpan = mock(Span.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, null); + defaultSpanScope.addSpanAttribute("key", 1.0); + + verify(mockSpan).addAttribute("key", 1.0); + } + + public void testAddSpanAttributeBoolean() { + Span mockSpan = mock(Span.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, null); + defaultSpanScope.addSpanAttribute("key", true); + + verify(mockSpan).addAttribute("key", true); + } + + public void testAddEvent() { + Span mockSpan = mock(Span.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, null); + defaultSpanScope.addSpanEvent("eventName"); + + verify(mockSpan).addEvent("eventName"); + } + + public void testSetError() { + Span mockSpan = mock(Span.class); + DefaultSpanScope defaultSpanScope = new DefaultSpanScope(mockSpan, null); + Exception ex = new Exception("error"); + defaultSpanScope.setError(ex); + + verify(mockSpan).setError(ex); + } + +} diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java index f0e8f3c2e2344..2b7a379b0051a 100644 --- a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java +++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java @@ -45,57 +45,12 @@ public void testCreateSpan() { public void testEndSpanByClosingScope() { DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - try (Scope scope = defaultTracer.startSpan("span_name")) { + try (SpanScope spanScope = defaultTracer.startSpan("span_name")) { verify(mockTracerContextStorage).put(TracerContextStorage.CURRENT_SPAN, mockSpan); } verify(mockTracerContextStorage).put(TracerContextStorage.CURRENT_SPAN, mockParentSpan); } - public void testAddSpanAttributeString() { - Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - defaultTracer.startSpan("span_name"); - - defaultTracer.addSpanAttribute("key", "value"); - - verify(mockSpan).addAttribute("key", "value"); - } - - public void testAddSpanAttributeLong() { - Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - defaultTracer.startSpan("span_name"); - - defaultTracer.addSpanAttribute("key", 1L); - - verify(mockSpan).addAttribute("key", 1L); - } - - public void testAddSpanAttributeDouble() { - Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - defaultTracer.startSpan("span_name"); - - defaultTracer.addSpanAttribute("key", 1.0); - - verify(mockSpan).addAttribute("key", 1.0); - } - - public void testAddSpanAttributeBoolean() { - Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - defaultTracer.startSpan("span_name"); - - defaultTracer.addSpanAttribute("key", true); - - verify(mockSpan).addAttribute("key", true); - } - - public void testAddEvent() { - Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - defaultTracer.startSpan("span_name"); - - defaultTracer.addSpanEvent("eventName"); - - verify(mockSpan).addEvent("eventName"); - } - public void testClose() throws IOException { Tracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java index 23a2d9baa3e6e..ba63df4ae47a1 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java @@ -9,6 +9,7 @@ package org.opensearch.telemetry.tracing; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; /** * Default implementation of {@link Span} using Otel span. It keeps a reference of OpenTelemetry Span and handles span @@ -48,6 +49,11 @@ public void addAttribute(String key, Boolean value) { delegateSpan.setAttribute(key, value); } + @Override + public void setError(Exception exception) { + delegateSpan.setStatus(StatusCode.ERROR, exception.getMessage()); + } + @Override public void addEvent(String event) { delegateSpan.addEvent(event); diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java index 7968c6c43afb4..df9cdd6669d23 100644 --- a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java +++ b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java @@ -45,7 +45,7 @@ public void testGetTracerWithTracingDisabledReturnsNoopTracer() { Tracer tracer = tracerFactory.getTracer(); assertTrue(tracer instanceof NoopTracer); - assertTrue(tracer.startSpan("foo") == Scope.NO_OP); + assertTrue(tracer.startSpan("foo") == SpanScope.NO_OP); } public void testGetTracerWithTracingEnabledReturnsDefaultTracer() { diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java index 4779d9796e11e..876145f6bf653 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java @@ -142,6 +142,10 @@ public Long getEndTime() { return endTime; } + public void setError(Exception exception) { + putMetadata("ERROR", exception.getMessage()); + } + private static class IdGenerator { private static String generateSpanId() { long id = randomSupplier.get().nextLong(); From 9e4e54d1972e359b5cbe5323c5c7220fd48be389 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 09:26:38 -0700 Subject: [PATCH 050/186] Bump io.projectreactor.netty:reactor-netty from 1.1.7 to 1.1.8 in /plugins/repository-azure (#8405) * Bump io.projectreactor.netty:reactor-netty in /plugins/repository-azure Bumps [io.projectreactor.netty:reactor-netty](https://github.com/reactor/reactor-netty) from 1.1.7 to 1.1.8. - [Release notes](https://github.com/reactor/reactor-netty/releases) - [Commits](https://github.com/reactor/reactor-netty/compare/v1.1.7...v1.1.8) --- updated-dependencies: - dependency-name: io.projectreactor.netty:reactor-netty dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] * Update changelog entry. Signed-off-by: Rishikesh1159 * Update changelog entry. Signed-off-by: Rishikesh1159 --------- Signed-off-by: dependabot[bot] Signed-off-by: Rishikesh1159 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Rishikesh1159 --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 4 ++-- .../repository-azure/licenses/reactor-netty-1.1.7.jar.sha1 | 1 - .../repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 | 1 + .../licenses/reactor-netty-core-1.1.7.jar.sha1 | 1 - .../licenses/reactor-netty-core-1.1.8.jar.sha1 | 1 + 6 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 plugins/repository-azure/licenses/reactor-netty-1.1.7.jar.sha1 create mode 100644 plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.7.jar.sha1 create mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index bb571cc21cd5c..99faefda9ad87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276)) - Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434)) - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) +- Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index e67ea7ab0a11e..4edb9e0b1913e 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -58,8 +58,8 @@ dependencies { api 'com.azure:azure-storage-blob:12.22.2' api 'org.reactivestreams:reactive-streams:1.0.4' api 'io.projectreactor:reactor-core:3.5.6' - api 'io.projectreactor.netty:reactor-netty:1.1.7' - api 'io.projectreactor.netty:reactor-netty-core:1.1.7' + api 'io.projectreactor.netty:reactor-netty:1.1.8' + api 'io.projectreactor.netty:reactor-netty-core:1.1.8' api 'io.projectreactor.netty:reactor-netty-http:1.1.8' api "org.slf4j:slf4j-api:${versions.slf4j}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" diff --git a/plugins/repository-azure/licenses/reactor-netty-1.1.7.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.1.7.jar.sha1 deleted file mode 100644 index 01a9b1d34d52f..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-1.1.7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c16497c29f96ea7b1db538cb0ddde55d9be173fe \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 new file mode 100644 index 0000000000000..6b6bf1903b16c --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 @@ -0,0 +1 @@ +d53a9d7d0395285f4c81664494fcd61477626e32 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.7.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.7.jar.sha1 deleted file mode 100644 index 62ed795cb11e9..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-core-1.1.7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d38bb526a501f52c4476b03730c710a96f8fd35b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1 new file mode 100644 index 0000000000000..707631f4dfe0c --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1 @@ -0,0 +1 @@ +48999c4ae27cdcee5eaff9dfd150a8b64624f0f5 \ No newline at end of file From d8cc450c88827f27a918473d320973b042965313 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:43:17 -0700 Subject: [PATCH 051/186] Fix flaky test. (#8441) Signed-off-by: Rishikesh1159 --- .../org/opensearch/indices/replication/SegmentReplicationIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index ce5e0989b622f..86794dd5ee811 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -435,6 +435,7 @@ public void testNodeDropWithOngoingReplication() throws Exception { refresh(INDEX_NAME); blockFileCopy.countDown(); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + ensureYellow(INDEX_NAME); assertBusy(() -> { assertDocCounts(docCount, replicaNode); }); state = client().admin().cluster().prepareState().execute().actionGet().getState(); // replica now promoted as primary should have same allocation id From 50d8006d5ffb72566b893ef23748d48b26c3a65e Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:55:53 -0400 Subject: [PATCH 052/186] Support transport action names when registering NamedRoutes (#7957) * Adds ability to register legacy action names along with route Signed-off-by: Darshit Chanpura Signed-off-by: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> --- CHANGELOG.md | 3 +- .../org/opensearch/action/ActionModule.java | 41 +++-- .../rest/RestInitializeExtensionAction.java | 4 +- .../rest/RestSendToExtensionAction.java | 42 +++-- .../extensions/rest/RouteHandler.java | 69 -------- .../java/org/opensearch/rest/NamedRoute.java | 156 ++++++++++++++++-- .../action/DynamicActionRegistryTests.java | 31 +++- .../extensions/ExtensionsManagerTests.java | 4 +- .../rest/RestSendToExtensionActionTests.java | 109 ++++++++++-- .../extensions/rest/RouteHandlerTests.java | 36 ---- .../org/opensearch/rest/NamedRouteTests.java | 102 ++++++++++-- 11 files changed, 411 insertions(+), 186 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/extensions/rest/RouteHandler.java delete mode 100644 server/src/test/java/org/opensearch/extensions/rest/RouteHandlerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 99faefda9ad87..aee081167d68f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) - [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) - Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) +- Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) @@ -169,4 +170,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 902ae7cc54e3f..56d1758161ced 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -462,9 +462,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; @@ -1044,7 +1044,7 @@ public static class DynamicActionRegistry { // A dynamic registry to add or remove Route / RestSendToExtensionAction pairs // at times other than node bootstrap. - private final Map routeRegistry = new ConcurrentHashMap<>(); + private final Map routeRegistry = new ConcurrentHashMap<>(); private final Set registeredActionNames = new ConcurrentSkipListSet<>(); @@ -1112,26 +1112,37 @@ public boolean isActionRegistered(String actionName) { } /** - * Add a dynamic action to the registry. + * Adds a dynamic route to the registry. * * @param route The route instance to add * @param action The corresponding instance of RestSendToExtensionAction to execute */ - public void registerDynamicRoute(RestHandler.Route route, RestSendToExtensionAction action) { + public void registerDynamicRoute(NamedRoute route, RestSendToExtensionAction action) { requireNonNull(route, "route is required"); requireNonNull(action, "action is required"); - Optional routeName = Optional.empty(); - if (route instanceof NamedRoute) { - routeName = Optional.of(((NamedRoute) route).name()); - if (isActionRegistered(routeName.get()) || registeredActionNames.contains(routeName.get())) { - throw new IllegalArgumentException("route [" + route + "] already registered"); - } + + String routeName = route.name(); + requireNonNull(routeName, "route name is required"); + if (isActionRegistered(routeName)) { + throw new IllegalArgumentException("route [" + route + "] already registered"); } + + Set actionNames = route.actionNames(); + if (!Collections.disjoint(actionNames, registeredActionNames)) { + Set alreadyRegistered = new HashSet<>(registeredActionNames); + alreadyRegistered.retainAll(actionNames); + String acts = String.join(", ", alreadyRegistered); + throw new IllegalArgumentException( + "action" + (alreadyRegistered.size() > 1 ? "s [" : " [") + acts + "] already registered" + ); + } + if (routeRegistry.containsKey(route)) { throw new IllegalArgumentException("route [" + route + "] already registered"); } routeRegistry.put(route, action); - routeName.ifPresent(registeredActionNames::add); + registeredActionNames.add(routeName); + registeredActionNames.addAll(actionNames); } /** @@ -1139,14 +1150,14 @@ public void registerDynamicRoute(RestHandler.Route route, RestSendToExtensionAct * * @param route The route to remove */ - public void unregisterDynamicRoute(RestHandler.Route route) { + public void unregisterDynamicRoute(NamedRoute route) { requireNonNull(route, "route is required"); if (routeRegistry.remove(route) == null) { throw new IllegalArgumentException("action [" + route + "] was not registered"); } - if (route instanceof NamedRoute) { - registeredActionNames.remove(((NamedRoute) route).name()); - } + + registeredActionNames.remove(route.name()); + registeredActionNames.removeAll(route.actionNames()); } /** diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index f47f342617732..878673b77a4a9 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -22,6 +22,7 @@ import org.opensearch.extensions.ExtensionsSettings.Extension; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; import org.opensearch.transport.ConnectTransportException; @@ -54,7 +55,7 @@ public String getName() { @Override public List routes() { - return List.of(new Route(POST, "/_extensions/initialize")); + return List.of(new NamedRoute.Builder().method(POST).path("/_extensions/initialize").uniqueName("extensions:initialize").build()); } public RestInitializeExtensionAction(ExtensionsManager extensionsManager) { @@ -187,6 +188,5 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client channel.sendResponse(new BytesRestResponse(RestStatus.ACCEPTED, builder)); } }; - } } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index 073b3f3f45818..3dd6056bb36cf 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -33,9 +33,9 @@ import java.nio.charset.StandardCharsets; import java.security.Principal; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.concurrent.CompletableFuture; @@ -90,33 +90,43 @@ public RestSendToExtensionAction( List restActionsAsRoutes = new ArrayList<>(); for (String restAction : restActionsRequest.getRestActions()) { - Optional name = Optional.empty(); + + // TODO Find a better way to parse these to avoid code-smells + + String name; + Set actionNames = new HashSet<>(); String[] parts = restAction.split(" "); - if (parts.length < 2) { - throw new IllegalArgumentException("REST action must contain at least a REST method and route"); + if (parts.length < 3) { + throw new IllegalArgumentException("REST action must contain at least a REST method, a route and a unique name"); } try { method = RestRequest.Method.valueOf(parts[0].trim()); path = pathPrefix + parts[1].trim(); - if (parts.length > 2) { - name = Optional.of(parts[2].trim()); + name = parts[2].trim(); + + // comma-separated action names + if (parts.length > 3) { + String[] actions = parts[3].split(","); + for (String action : actions) { + String trimmed = action.trim(); + if (!trimmed.isEmpty()) { + actionNames.add(trimmed); + } + } } } catch (IndexOutOfBoundsException | IllegalArgumentException e) { throw new IllegalArgumentException(restAction + " does not begin with a valid REST method"); } - logger.info("Registering: " + method + " " + path); - if (name.isPresent()) { - NamedRoute nr = new NamedRoute(method, path, name.get()); - restActionsAsRoutes.add(nr); - dynamicActionRegistry.registerDynamicRoute(nr, this); - } else { - Route r = new Route(method, path); - restActionsAsRoutes.add(r); - dynamicActionRegistry.registerDynamicRoute(r, this); - } + logger.info("Registering: " + method + " " + path + " " + name); + + // All extension routes being registered must have a unique name associated with them + NamedRoute nr = new NamedRoute.Builder().method(method).path(path).uniqueName(name).legacyActionNames(actionNames).build(); + restActionsAsRoutes.add(nr); + dynamicActionRegistry.registerDynamicRoute(nr, this); } this.routes = unmodifiableList(restActionsAsRoutes); + // TODO: Modify {@link NamedRoute} to support deprecated route registration List restActionsAsDeprecatedRoutes = new ArrayList<>(); // Iterate in pairs of route / deprecation message List deprecatedActions = restActionsRequest.getDeprecatedRestActions(); diff --git a/server/src/main/java/org/opensearch/extensions/rest/RouteHandler.java b/server/src/main/java/org/opensearch/extensions/rest/RouteHandler.java deleted file mode 100644 index 189d67c120189..0000000000000 --- a/server/src/main/java/org/opensearch/extensions/rest/RouteHandler.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.extensions.rest; - -import java.util.function.Function; - -import org.opensearch.rest.RestHandler.Route; -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestRequest.Method; - -/** - * A subclass of {@link Route} that includes a handler method for that route. - */ -public class RouteHandler extends Route { - - private final String name; - - private final Function responseHandler; - - /** - * Handle the method and path with the specified handler. - * - * @param method The {@link Method} to handle. - * @param path The path to handle. - * @param handler The method which handles the method and path. - */ - public RouteHandler(Method method, String path, Function handler) { - super(method, path); - this.responseHandler = handler; - this.name = null; - } - - /** - * Handle the method and path with the specified handler. - * - * @param name The name of the handler. - * @param method The {@link Method} to handle. - * @param path The path to handle. - * @param handler The method which handles the method and path. - */ - public RouteHandler(String name, Method method, String path, Function handler) { - super(method, path); - this.responseHandler = handler; - this.name = name; - } - - /** - * Executes the handler for this route. - * - * @param request The request to handle - * @return the {@link ExtensionRestResponse} result from the handler for this route. - */ - public ExtensionRestResponse handleRequest(RestRequest request) { - return responseHandler.apply(request); - } - - /** - * The name of the RouteHandler. Must be unique across route handlers. - */ - public String name() { - return this.name; - } -} diff --git a/server/src/main/java/org/opensearch/rest/NamedRoute.java b/server/src/main/java/org/opensearch/rest/NamedRoute.java index f5eaafcd04056..109f688a4924e 100644 --- a/server/src/main/java/org/opensearch/rest/NamedRoute.java +++ b/server/src/main/java/org/opensearch/rest/NamedRoute.java @@ -9,6 +9,13 @@ package org.opensearch.rest; import org.opensearch.OpenSearchException; +import org.opensearch.transport.TransportService; + +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; /** * A named Route @@ -16,21 +23,123 @@ * @opensearch.internal */ public class NamedRoute extends RestHandler.Route { + private static final String VALID_ACTION_NAME_PATTERN = "^[a-zA-Z0-9:/*_]*$"; static final int MAX_LENGTH_OF_ACTION_NAME = 250; - private final String name; + private final String uniqueName; + private final Set actionNames; - public boolean isValidRouteName(String routeName) { - if (routeName == null || routeName.isBlank() || routeName.length() > MAX_LENGTH_OF_ACTION_NAME) { - return false; + private Function handler; + + /** + * Builder class for constructing instances of {@link NamedRoute}. + */ + public static class Builder { + private RestRequest.Method method; + private String path; + private String uniqueName; + private final Set legacyActionNames = new HashSet<>(); + private Function handler; + + /** + * Sets the REST method for the route. + * + * @param method the REST method for the route + * @return the builder instance + */ + public Builder method(RestRequest.Method method) { + requireNonNull(method, "REST method must not be null."); + this.method = method; + return this; + } + + /** + * Sets the URL path for the route. + * + * @param path the URL path for the route + * @return the builder instance + */ + public Builder path(String path) { + requireNonNull(path, "REST path must not be null."); + this.path = path; + return this; + } + + /** + * Sets the name for the route. + * + * @param name the name for the route + * @return the builder instance + */ + public Builder uniqueName(String name) { + requireNonNull(name, "REST route name must not be null."); + this.uniqueName = name; + return this; + } + + /** + * Sets the legacy action names for the route. + * + * @param legacyActionNames the legacy action names for the route + * @return the builder instance + */ + public Builder legacyActionNames(Set legacyActionNames) { + this.legacyActionNames.addAll(validateLegacyActionNames(legacyActionNames)); + return this; + } + + /** + * Sets the handler for this route + * + * @param handler the handler for this route + * @return the builder instance + */ + public Builder handler(Function handler) { + requireNonNull(handler, "Route handler must not be null."); + this.handler = handler; + return this; + } + + /** + * Builds a new instance of {@link NamedRoute} based on the provided parameters. + * + * @return a new instance of {@link NamedRoute} + * @throws OpenSearchException if the route name is invalid + */ + public NamedRoute build() { + checkIfFieldsAreSet(); + return new NamedRoute(this); + } + + /** + * Checks if all builder fields are set before creating a new NamedRoute object + */ + private void checkIfFieldsAreSet() { + if (method == null || path == null || uniqueName == null) { + throw new IllegalStateException("REST method, path and uniqueName are required."); + } + } + + private Set validateLegacyActionNames(Set legacyActionNames) { + if (legacyActionNames == null) { + return new HashSet<>(); + } + for (String actionName : legacyActionNames) { + if (!TransportService.isValidActionName(actionName)) { + throw new OpenSearchException( + "Invalid action name [" + actionName + "]. It must start with one of: " + TransportService.VALID_ACTION_PREFIXES + ); + } + } + return legacyActionNames; } - return routeName.matches(VALID_ACTION_NAME_PATTERN); + } - public NamedRoute(RestRequest.Method method, String path, String name) { - super(method, path); - if (!isValidRouteName(name)) { + private NamedRoute(Builder builder) { + super(builder.method, builder.path); + if (!isValidRouteName(builder.uniqueName)) { throw new OpenSearchException( "Invalid route name specified. The route name may include the following characters" + " 'a-z', 'A-Z', '0-9', ':', '/', '*', '_' and be less than " @@ -38,18 +147,43 @@ public NamedRoute(RestRequest.Method method, String path, String name) { + " characters" ); } - this.name = name; + this.uniqueName = builder.uniqueName; + this.actionNames = Set.copyOf(builder.legacyActionNames); + this.handler = builder.handler; + } + + public boolean isValidRouteName(String routeName) { + return routeName != null + && !routeName.isBlank() + && routeName.length() <= MAX_LENGTH_OF_ACTION_NAME + && routeName.matches(VALID_ACTION_NAME_PATTERN); } /** * The name of the Route. Must be unique across Route. */ public String name() { - return this.name; + return this.uniqueName; + } + + /** + * The legacy transport Action name to match against this route to support authorization in REST layer. + * MUST be unique across all Routes + */ + public Set actionNames() { + return this.actionNames; + } + + /** + * The handler associated with this route + * @return the handler associated with this route + */ + public Function handler() { + return handler; } @Override public String toString() { - return "NamedRoute [method=" + method + ", path=" + path + ", name=" + name + "]"; + return "NamedRoute [method=" + method + ", path=" + path + ", name=" + uniqueName + ", actionNames=" + actionNames + "]"; } } diff --git a/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java b/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java index 963d47df3baff..17e424ee81e7e 100644 --- a/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java +++ b/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import java.util.Set; import static org.mockito.Mockito.mock; @@ -80,8 +81,8 @@ public void testDynamicActionRegistry() { public void testDynamicActionRegistryWithNamedRoutes() { RestSendToExtensionAction action = mock(RestSendToExtensionAction.class); RestSendToExtensionAction action2 = mock(RestSendToExtensionAction.class); - NamedRoute r1 = new NamedRoute(RestRequest.Method.GET, "/foo", "foo"); - NamedRoute r2 = new NamedRoute(RestRequest.Method.GET, "/bar", "bar"); + NamedRoute r1 = new NamedRoute.Builder().method(RestRequest.Method.GET).path("/foo").uniqueName("foo").build(); + NamedRoute r2 = new NamedRoute.Builder().method(RestRequest.Method.PUT).path("/bar").uniqueName("bar").build(); DynamicActionRegistry registry = new DynamicActionRegistry(); registry.registerDynamicRoute(r1, action); @@ -89,22 +90,38 @@ public void testDynamicActionRegistryWithNamedRoutes() { assertTrue(registry.isActionRegistered("foo")); assertTrue(registry.isActionRegistered("bar")); + + registry.unregisterDynamicRoute(r2); + + assertTrue(registry.isActionRegistered("foo")); + assertFalse(registry.isActionRegistered("bar")); } - public void testDynamicActionRegistryRegisterAndUnregisterWithNamedRoutes() { + public void testDynamicActionRegistryWithNamedRoutesAndLegacyActionNames() { RestSendToExtensionAction action = mock(RestSendToExtensionAction.class); RestSendToExtensionAction action2 = mock(RestSendToExtensionAction.class); - NamedRoute r1 = new NamedRoute(RestRequest.Method.GET, "/foo", "foo"); - NamedRoute r2 = new NamedRoute(RestRequest.Method.GET, "/bar", "bar"); + NamedRoute r1 = new NamedRoute.Builder().method(RestRequest.Method.GET) + .path("/foo") + .uniqueName("foo") + .legacyActionNames(Set.of("cluster:admin/opensearch/abc/foo")) + .build(); + NamedRoute r2 = new NamedRoute.Builder().method(RestRequest.Method.PUT) + .path("/bar") + .uniqueName("bar") + .legacyActionNames(Set.of("cluster:admin/opensearch/xyz/bar")) + .build(); DynamicActionRegistry registry = new DynamicActionRegistry(); registry.registerDynamicRoute(r1, action); registry.registerDynamicRoute(r2, action2); + assertTrue(registry.isActionRegistered("cluster:admin/opensearch/abc/foo")); + assertTrue(registry.isActionRegistered("cluster:admin/opensearch/xyz/bar")); + registry.unregisterDynamicRoute(r2); - assertTrue(registry.isActionRegistered("foo")); - assertFalse(registry.isActionRegistered("bar")); + assertTrue(registry.isActionRegistered("cluster:admin/opensearch/abc/foo")); + assertFalse(registry.isActionRegistered("cluster:admin/opensearch/xyz/bar")); } private static final class TestAction extends ActionType { diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 713a70c6a7d3e..3f61d01166fb9 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -443,8 +443,8 @@ public void testHandleRegisterRestActionsRequest() throws Exception { initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; - List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); - List deprecatedActionsList = List.of("GET /deprecated/foo", "It's deprecated!"); + List actionsList = List.of("GET /foo foo", "PUT /bar bar", "POST /baz baz"); + List deprecatedActionsList = List.of("GET /deprecated/foo foo_deprecated", "It's deprecated!"); RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest(uniqueIdStr, actionsList, deprecatedActionsList); TransportResponse response = extensionsManager.getRestActionsRequestHandler() .handleRegisterRestActionsRequest(registerActionsRequest, actionModule.getDynamicActionRegistry()); diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index fe8792b36f048..23a9169b91e21 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -135,8 +135,8 @@ public void tearDown() throws Exception { public void testRestSendToExtensionAction() throws Exception { RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( "uniqueid1", - List.of("GET /foo", "PUT /bar", "POST /baz"), - List.of("GET /deprecated/foo", "It's deprecated!") + List.of("GET /foo foo", "PUT /bar bar", "POST /baz baz"), + List.of("GET /deprecated/foo foo_deprecated", "Its deprecated") ); RestSendToExtensionAction restSendToExtensionAction = new RestSendToExtensionAction( registerRestActionRequest, @@ -180,9 +180,70 @@ public void testRestSendToExtensionActionWithNamedRoute() throws Exception { assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); List expected = new ArrayList<>(); String uriPrefix = "/_extensions/_uniqueid1"; - expected.add(new NamedRoute(Method.GET, uriPrefix + "/foo", "foo")); - expected.add(new NamedRoute(Method.PUT, uriPrefix + "/bar", "bar")); - expected.add(new NamedRoute(Method.POST, uriPrefix + "/baz", "baz")); + NamedRoute nr1 = new NamedRoute.Builder().method(Method.GET).path(uriPrefix + "/foo").uniqueName("foo").build(); + + NamedRoute nr2 = new NamedRoute.Builder().method(Method.PUT).path(uriPrefix + "/bar").uniqueName("bar").build(); + + NamedRoute nr3 = new NamedRoute.Builder().method(Method.POST).path(uriPrefix + "/baz").uniqueName("baz").build(); + + expected.add(nr1); + expected.add(nr2); + expected.add(nr3); + + List routes = restSendToExtensionAction.routes(); + assertEquals(expected.size(), routes.size()); + List expectedPaths = expected.stream().map(Route::getPath).collect(Collectors.toList()); + List paths = routes.stream().map(Route::getPath).collect(Collectors.toList()); + List expectedMethods = expected.stream().map(Route::getMethod).collect(Collectors.toList()); + List methods = routes.stream().map(Route::getMethod).collect(Collectors.toList()); + List expectedNames = expected.stream().map(NamedRoute::name).collect(Collectors.toList()); + List names = routes.stream().map(r -> ((NamedRoute) r).name()).collect(Collectors.toList()); + assertTrue(paths.containsAll(expectedPaths)); + assertTrue(expectedPaths.containsAll(paths)); + assertTrue(methods.containsAll(expectedMethods)); + assertTrue(expectedMethods.containsAll(methods)); + assertTrue(expectedNames.containsAll(names)); + } + + public void testRestSendToExtensionActionWithNamedRouteAndLegacyActionName() throws Exception { + RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( + "uniqueid1", + List.of( + "GET /foo foo cluster:admin/opensearch/abc/foo", + "PUT /bar bar cluster:admin/opensearch/jkl/bar,cluster:admin/opendistro/mno/bar*", + "POST /baz baz cluster:admin/opensearch/xyz/baz" + ), + List.of("GET /deprecated/foo foo_deprecated cluster:admin/opensearch/abc/foo_deprecated", "It's deprecated!") + ); + RestSendToExtensionAction restSendToExtensionAction = new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry + ); + + assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); + List expected = new ArrayList<>(); + String uriPrefix = "/_extensions/_uniqueid1"; + NamedRoute nr1 = new NamedRoute.Builder().method(Method.GET) + .path(uriPrefix + "/foo") + .uniqueName("foo") + .legacyActionNames(Set.of("cluster:admin/opensearch/abc/foo")) + .build(); + NamedRoute nr2 = new NamedRoute.Builder().method(Method.PUT) + .path(uriPrefix + "/bar") + .uniqueName("bar") + .legacyActionNames(Set.of("cluster:admin/opensearch/jkl/bar", "cluster:admin/opendistro/mno/bar*")) + .build(); + NamedRoute nr3 = new NamedRoute.Builder().method(Method.POST) + .path(uriPrefix + "/baz") + .uniqueName("baz") + .legacyActionNames(Set.of("cluster:admin/opensearch/xyz/baz")) + .build(); + + expected.add(nr1); + expected.add(nr2); + expected.add(nr3); List routes = restSendToExtensionAction.routes(); assertEquals(expected.size(), routes.size()); @@ -192,11 +253,26 @@ public void testRestSendToExtensionActionWithNamedRoute() throws Exception { List methods = routes.stream().map(Route::getMethod).collect(Collectors.toList()); List expectedNames = expected.stream().map(NamedRoute::name).collect(Collectors.toList()); List names = routes.stream().map(r -> ((NamedRoute) r).name()).collect(Collectors.toList()); + Set expectedActionNames = expected.stream().flatMap(nr -> nr.actionNames().stream()).collect(Collectors.toSet()); + Set actionNames = routes.stream().flatMap(nr -> ((NamedRoute) nr).actionNames().stream()).collect(Collectors.toSet()); assertTrue(paths.containsAll(expectedPaths)); assertTrue(expectedPaths.containsAll(paths)); assertTrue(methods.containsAll(expectedMethods)); assertTrue(expectedMethods.containsAll(methods)); assertTrue(expectedNames.containsAll(names)); + assertTrue(expectedActionNames.containsAll(actionNames)); + } + + public void testRestSendToExtensionActionWithoutUniqueNameShouldFail() { + RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( + "uniqueid1", + List.of("GET /foo", "PUT /bar"), + List.of() + ); + expectThrows( + IllegalArgumentException.class, + () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + ); } public void testRestSendToExtensionMultipleNamedRoutesWithSameName() throws Exception { @@ -211,6 +287,18 @@ public void testRestSendToExtensionMultipleNamedRoutesWithSameName() throws Exce ); } + public void testRestSendToExtensionMultipleNamedRoutesWithSameLegacyActionName() throws Exception { + RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( + "uniqueid1", + List.of("GET /foo foo cluster:admin/opensearch/abc/foo", "PUT /bar bar cluster:admin/opensearch/abc/foo"), + List.of() + ); + expectThrows( + IllegalArgumentException.class, + () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + ); + } + public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPath() throws Exception { RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( "uniqueid1", @@ -226,7 +314,7 @@ public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPath() throws public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPathWithDifferentPathParams() throws Exception { RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( "uniqueid1", - List.of("GET /foo/{path_param1}", "GET /foo/{path_param2}"), + List.of("GET /foo/{path_param1} fooWithParam", "GET /foo/{path_param2} listFooWithParam"), List.of() ); expectThrows( @@ -235,12 +323,13 @@ public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPathWithDiffer ); } - public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPathWithPathParams() throws Exception { + public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPathWithPathParams() { RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( "uniqueid1", - List.of("GET /foo/{path_param}", "GET /foo/{path_param}/list"), + List.of("GET /foo/{path_param} fooWithParam", "GET /foo/{path_param}/list listFooWithParam"), List.of() ); + try { new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry); } catch (IllegalArgumentException e) { @@ -285,8 +374,8 @@ public void testRestSendToExtensionWithNamedRouteCollidingWithNativeTransportAct public void testRestSendToExtensionActionFilterHeaders() throws Exception { RegisterRestActionsRequest registerRestActionRequest = new RegisterRestActionsRequest( "uniqueid1", - List.of("GET /foo", "PUT /bar", "POST /baz"), - List.of("GET /deprecated/foo", "It's deprecated!") + List.of("GET /foo foo", "PUT /bar bar", "POST /baz baz"), + List.of("GET /deprecated/foo foo_deprecated", "It's deprecated!") ); RestSendToExtensionAction restSendToExtensionAction = new RestSendToExtensionAction( registerRestActionRequest, diff --git a/server/src/test/java/org/opensearch/extensions/rest/RouteHandlerTests.java b/server/src/test/java/org/opensearch/extensions/rest/RouteHandlerTests.java deleted file mode 100644 index 855296b2038f0..0000000000000 --- a/server/src/test/java/org/opensearch/extensions/rest/RouteHandlerTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.extensions.rest; - -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; -import org.opensearch.test.OpenSearchTestCase; - -public class RouteHandlerTests extends OpenSearchTestCase { - public void testUnnamedRouteHandler() { - RouteHandler rh = new RouteHandler( - RestRequest.Method.GET, - "/foo/bar", - req -> new ExtensionRestResponse(req, RestStatus.OK, "content") - ); - - assertEquals(null, rh.name()); - } - - public void testNamedRouteHandler() { - RouteHandler rh = new RouteHandler( - "foo", - RestRequest.Method.GET, - "/foo/bar", - req -> new ExtensionRestResponse(req, RestStatus.OK, "content") - ); - - assertEquals("foo", rh.name()); - } -} diff --git a/server/src/test/java/org/opensearch/rest/NamedRouteTests.java b/server/src/test/java/org/opensearch/rest/NamedRouteTests.java index d489321ea5dc6..cf3e2b5b858bf 100644 --- a/server/src/test/java/org/opensearch/rest/NamedRouteTests.java +++ b/server/src/test/java/org/opensearch/rest/NamedRouteTests.java @@ -11,22 +11,17 @@ import org.opensearch.OpenSearchException; import org.opensearch.test.OpenSearchTestCase; +import java.util.Set; +import java.util.function.Function; + import static org.opensearch.rest.NamedRoute.MAX_LENGTH_OF_ACTION_NAME; +import static org.opensearch.rest.RestRequest.Method.GET; public class NamedRouteTests extends OpenSearchTestCase { - public void testNamedRouteWithNullName() { - try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", null); - fail("Expected NamedRoute to throw exception on null name provided"); - } catch (OpenSearchException e) { - assertTrue(e.getMessage().contains("Invalid route name specified")); - } - } - public void testNamedRouteWithEmptyName() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", ""); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("").build(); fail("Expected NamedRoute to throw exception on empty name provided"); } catch (OpenSearchException e) { assertTrue(e.getMessage().contains("Invalid route name specified")); @@ -35,7 +30,7 @@ public void testNamedRouteWithEmptyName() { public void testNamedRouteWithNameContainingSpace() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", "foo bar"); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo bar").build(); fail("Expected NamedRoute to throw exception on name containing space name provided"); } catch (OpenSearchException e) { assertTrue(e.getMessage().contains("Invalid route name specified")); @@ -44,7 +39,7 @@ public void testNamedRouteWithNameContainingSpace() { public void testNamedRouteWithNameContainingInvalidCharacters() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", "foo@bar!"); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo@bar!").build(); fail("Expected NamedRoute to throw exception on name containing invalid characters name provided"); } catch (OpenSearchException e) { assertTrue(e.getMessage().contains("Invalid route name specified")); @@ -54,7 +49,7 @@ public void testNamedRouteWithNameContainingInvalidCharacters() { public void testNamedRouteWithNameOverMaximumLength() { try { String repeated = new String(new char[MAX_LENGTH_OF_ACTION_NAME + 1]).replace("\0", "x"); - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", repeated); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName(repeated).build(); fail("Expected NamedRoute to throw exception on name over maximum length supplied"); } catch (OpenSearchException e) { assertTrue(e.getMessage().contains("Invalid route name specified")); @@ -63,7 +58,7 @@ public void testNamedRouteWithNameOverMaximumLength() { public void testNamedRouteWithValidActionName() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", "foo:bar"); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar").build(); } catch (OpenSearchException e) { fail("Did not expect NamedRoute to throw exception on valid action name"); } @@ -71,7 +66,7 @@ public void testNamedRouteWithValidActionName() { public void testNamedRouteWithValidActionNameWithForwardSlash() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", "foo:bar/baz"); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar:baz").build(); } catch (OpenSearchException e) { fail("Did not expect NamedRoute to throw exception on valid action name"); } @@ -79,7 +74,7 @@ public void testNamedRouteWithValidActionNameWithForwardSlash() { public void testNamedRouteWithValidActionNameWithWildcard() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", "foo:bar/*"); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar/*").build(); } catch (OpenSearchException e) { fail("Did not expect NamedRoute to throw exception on valid action name"); } @@ -87,9 +82,82 @@ public void testNamedRouteWithValidActionNameWithWildcard() { public void testNamedRouteWithValidActionNameWithUnderscore() { try { - NamedRoute r = new NamedRoute(RestRequest.Method.GET, "foo/bar", "foo:bar_baz"); + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar_baz").build(); } catch (OpenSearchException e) { fail("Did not expect NamedRoute to throw exception on valid action name"); } } + + public void testNamedRouteWithNullLegacyActionNames() { + try { + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar").legacyActionNames(null).build(); + assertTrue(r.actionNames().isEmpty()); + } catch (OpenSearchException e) { + fail("Did not expect NamedRoute to pass with an invalid legacy action name"); + } + } + + public void testNamedRouteWithInvalidLegacyActionNames() { + try { + NamedRoute r = new NamedRoute.Builder().method(GET) + .path("foo/bar") + .uniqueName("foo:bar") + .legacyActionNames(Set.of("foo:bar-legacy")) + .build(); + fail("Did not expect NamedRoute to pass with an invalid legacy action name"); + } catch (OpenSearchException e) { + assertTrue(e.getMessage().contains("Invalid action name [foo:bar-legacy]. It must start with one of:")); + } + } + + public void testNamedRouteWithHandler() { + Function fooHandler = restRequest -> null; + try { + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar_baz").handler(fooHandler).build(); + assertEquals(r.handler(), fooHandler); + } catch (OpenSearchException e) { + fail("Did not expect NamedRoute to throw exception"); + } + } + + public void testNamedRouteNullChecks() { + try { + NamedRoute r = new NamedRoute.Builder().method(null).path("foo/bar").uniqueName("foo:bar_baz").build(); + fail("Expected NamedRoute to throw exception as method should not be null"); + } catch (NullPointerException e) { + assertEquals("REST method must not be null.", e.getMessage()); + } + + try { + NamedRoute r = new NamedRoute.Builder().method(GET).path(null).uniqueName("foo:bar_baz").build(); + fail("Expected NamedRoute to throw exception as path should not be null"); + } catch (NullPointerException e) { + assertEquals("REST path must not be null.", e.getMessage()); + } + + try { + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName(null).build(); + fail("Expected NamedRoute to throw exception as route name should not be null"); + } catch (NullPointerException e) { + assertEquals("REST route name must not be null.", e.getMessage()); + } + + try { + NamedRoute r = new NamedRoute.Builder().method(GET).path("foo/bar").uniqueName("foo:bar_baz").handler(null).build(); + fail("Expected NamedRoute to throw exception as handler should not be null"); + } catch (NullPointerException e) { + assertEquals("Route handler must not be null.", e.getMessage()); + } + } + + public void testNamedRouteEmptyBuild() { + try { + NamedRoute r = new NamedRoute.Builder().build(); + fail("Expected NamedRoute to throw exception as fields should not be null"); + } catch (IllegalStateException e) { + assertEquals("REST method, path and uniqueName are required.", e.getMessage()); + } + + } + } From 4657fe70502631a815ca18888700808145ba0523 Mon Sep 17 00:00:00 2001 From: Mingshi Liu <113382730+mingshl@users.noreply.github.com> Date: Wed, 5 Jul 2023 15:47:44 -0700 Subject: [PATCH 053/186] Changing version check to 2.9 (#8416) Update the version of BWC test from 3.0 to 2.9 for search pipeline statistic after the PR https://github.com/opensearch-project/OpenSearch/pull/8053 / commit https://github.com/opensearch-project/OpenSearch/commit/46c9a211b6b9490f6a7ac9425e946986cd51bed2 backported to 2.x branch. Signed-off-by: Mingshi Liu --- .../opensearch/action/admin/cluster/node/stats/NodeStats.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index 6b8e06594acb7..4cdc54f9c7952 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -193,7 +193,7 @@ public NodeStats(StreamInput in) throws IOException { } else { taskCancellationStats = null; } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { // TODO Update to 2_9_0 when we backport to 2.x + if (in.getVersion().onOrAfter(Version.V_2_9_0)) { searchPipelineStats = in.readOptionalWriteable(SearchPipelineStats::new); } else { searchPipelineStats = null; @@ -427,7 +427,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeOptionalWriteable(taskCancellationStats); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { // TODO: Update to 2_9_0 once we backport to 2.x + if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeOptionalWriteable(searchPipelineStats); } } From 8bebf5b9efe7a6f6668a69d4c156d42f30b99332 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Wed, 5 Jul 2023 18:02:23 -0700 Subject: [PATCH 054/186] Unmute remote store test testPressureServiceStats (#8458) Signed-off-by: Kunal Kotwani --- .../remotestore/SegmentReplicationUsingRemoteStoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index 01fb91f83aa02..d39b30ada5ef7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -64,7 +64,6 @@ public void teardown() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7592") @Override public void testPressureServiceStats() throws Exception { super.testPressureServiceStats(); From eca5a6cbc4f8454afaebd35e81f4d3c23071d92d Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Thu, 6 Jul 2023 14:01:31 +0530 Subject: [PATCH 055/186] default compression level change (#8471) Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 1 + .../index/codec/customcodecs/Lucene95CustomCodec.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aee081167d68f..619bb0d58d8ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652)) - Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963)) - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) +- Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java index 3c570f9d0566c..8aa422a47a073 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java @@ -23,7 +23,7 @@ * @opensearch.internal */ public abstract class Lucene95CustomCodec extends FilterCodec { - public static final int DEFAULT_COMPRESSION_LEVEL = 6; + public static final int DEFAULT_COMPRESSION_LEVEL = 3; /** Each mode represents a compression algorithm. */ public enum Mode { From c1c23b42e335bb337c668c9cf9dccd8b71dfdbab Mon Sep 17 00:00:00 2001 From: Ashish Date: Thu, 6 Jul 2023 14:09:41 +0530 Subject: [PATCH 056/186] Add logs to debug NoSuchFileException during segments upload (#8475) Signed-off-by: Ashish Singh --- .../index/shard/RemoteStoreRefreshListener.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index ddca12d9283f3..4eef02eeaf380 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -224,7 +224,17 @@ private synchronized void syncSegments(boolean isRetry) { // Each metadata file in the remote segment store represents a commit and the following // statement keeps sure that each metadata will always contain all the segments from last commit + refreshed // segments. - localSegmentsPostRefresh.addAll(SegmentInfos.readCommit(storeDirectory, latestSegmentInfos.get()).files(true)); + SegmentInfos segmentCommitInfos; + try { + segmentCommitInfos = SegmentInfos.readCommit(storeDirectory, latestSegmentInfos.get()); + } catch (Exception e) { + // Seeing discrepancy in segment infos and files on disk. SegmentInfosSnapshot is returning + // a segment_N file which does not exist on local disk. + logger.error("Exception occurred while SegmentInfos.readCommit(..)", e); + logger.error("segmentInfosFiles={} diskFiles={}", localSegmentsPostRefresh, storeDirectory.listAll()); + throw e; + } + localSegmentsPostRefresh.addAll(segmentCommitInfos.files(true)); segmentInfosFiles.stream() .filter(file -> !file.equals(latestSegmentInfos.get())) .forEach(localSegmentsPostRefresh::remove); From 5c0e3c94950796f7c99a20456528181ae9f16bbf Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 6 Jul 2023 19:46:36 +0530 Subject: [PATCH 057/186] [Remote Segment Store] Make metadata file immutable (#8363) Authored-by: Sachin Kale --- .../replication/SegmentReplicationIT.java | 1 + .../opensearch/remotestore/RemoteStoreIT.java | 2 +- .../shard/RemoteStoreRefreshListener.java | 20 +- .../index/store/RemoteDirectory.java | 41 +++ .../store/RemoteSegmentStoreDirectory.java | 167 ++++----- .../translog/InternalTranslogManager.java | 5 + .../index/translog/NoOpTranslogManager.java | 5 + .../index/translog/TranslogManager.java | 2 + .../blobstore/fs/FsBlobContainerTests.java | 2 +- .../index/store/RemoteDirectoryTests.java | 57 +++- ...moteSegmentStoreDirectoryFactoryTests.java | 20 +- .../RemoteSegmentStoreDirectoryTests.java | 318 ++++++++---------- 12 files changed, 351 insertions(+), 289 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 86794dd5ee811..2b879f1c37d88 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -800,6 +800,7 @@ public void testReplicaHasDiffFilesThanPrimary() throws Exception { public void testPressureServiceStats() throws Exception { final String primaryNode = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); + ensureYellow(INDEX_NAME); final String replicaNode = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 77de601b53ec6..be8976671d04f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -320,6 +320,6 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { .get() .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID, "/0/segments/metadata"); - assertEquals(1, getFileCount(indexPath)); + assertEquals(numberOfIterations, getFileCount(indexPath)); } } diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 4eef02eeaf380..cd3e7aa3b11a9 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -30,6 +30,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; +import org.opensearch.index.translog.Translog; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.threadpool.Scheduler; @@ -359,12 +360,19 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo)); segmentInfosSnapshot.setUserData(userData, false); - remoteDirectory.uploadMetadata( - localSegmentsPostRefresh, - segmentInfosSnapshot, - storeDirectory, - indexShard.getOperationPrimaryTerm() - ); + Translog.TranslogGeneration translogGeneration = indexShard.getEngine().translogManager().getTranslogGeneration(); + if (translogGeneration == null) { + throw new UnsupportedOperationException("Encountered null TranslogGeneration while uploading metadata to remote segment store"); + } else { + long translogFileGeneration = translogGeneration.translogFileGeneration; + remoteDirectory.uploadMetadata( + localSegmentsPostRefresh, + segmentInfosSnapshot, + storeDirectory, + indexShard.getOperationPrimaryTerm(), + translogFileGeneration + ); + } } private boolean uploadNewSegments(Collection localSegmentsPostRefresh) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index be4b4e910bb4d..8782808c070ab 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -13,6 +13,8 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; +import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -20,10 +22,15 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; /** * A {@code RemoteDirectory} provides an abstraction layer for storing a list of files to a remote store. @@ -61,6 +68,40 @@ public Collection listFilesByPrefix(String filenamePrefix) throws IOExce return blobContainer.listBlobsByPrefix(filenamePrefix).keySet(); } + public List listFilesByPrefixInLexicographicOrder(String filenamePrefix, int limit) throws IOException { + List sortedBlobList = new ArrayList<>(); + AtomicReference exception = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + LatchedActionListener> actionListener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + sortedBlobList.addAll(blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList())); + } + + @Override + public void onFailure(Exception e) { + exception.set(e); + } + }, latch); + + try { + blobContainer.listBlobsByPrefixInSortedOrder( + filenamePrefix, + limit, + BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC, + actionListener + ); + latch.await(); + } catch (InterruptedException e) { + throw new IOException("Exception in listFilesByPrefixInLexicographicOrder with prefix: " + filenamePrefix, e); + } + if (exception.get() != null) { + throw new IOException(exception.get()); + } else { + return sortedBlobList; + } + } + /** * Removes an existing file in the directory. * diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index ac129aca8baf7..e7602203440d2 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -22,6 +22,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.store.lockmanager.FileLockInfo; import org.opensearch.index.store.lockmanager.RemoteStoreCommitLevelLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; @@ -34,15 +35,14 @@ import java.nio.file.NoSuchFileException; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; /** @@ -62,9 +62,6 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement */ public static final String SEGMENT_NAME_UUID_SEPARATOR = "__"; - public static final MetadataFilenameUtils.MetadataFilenameComparator METADATA_FILENAME_COMPARATOR = - new MetadataFilenameUtils.MetadataFilenameComparator(); - /** * remoteDataDirectory is used to store segment files at path: cluster_UUID/index_UUID/shardId/segments/data */ @@ -78,12 +75,6 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement private final ThreadPool threadPool; - /** - * To prevent explosion of refresh metadata files, we replace refresh files for the given primary term and generation - * This is achieved by uploading refresh metadata file with the same UUID suffix. - */ - private String commonFilenameSuffix; - /** * Keeps track of local segment filename to uploaded filename along with other attributes like checksum. * This map acts as a cache layer for uploaded segment filenames which helps avoid calling listAll() each time. @@ -105,6 +96,8 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement */ protected final AtomicBoolean canDeleteStaleCommits = new AtomicBoolean(true); + private final AtomicLong metadataUploadCounter = new AtomicLong(0); + public RemoteSegmentStoreDirectory( RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory, @@ -127,7 +120,6 @@ public RemoteSegmentStoreDirectory( * @throws IOException if there were any failures in reading the metadata file */ public RemoteSegmentMetadata init() throws IOException { - this.commonFilenameSuffix = UUIDs.base64UUID(); RemoteSegmentMetadata remoteSegmentMetadata = readLatestMetadataFile(); if (remoteSegmentMetadata != null) { this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(remoteSegmentMetadata.getMetadata()); @@ -170,12 +162,15 @@ public RemoteSegmentMetadata initializeToSpecificCommit(long primaryTerm, long c public RemoteSegmentMetadata readLatestMetadataFile() throws IOException { RemoteSegmentMetadata remoteSegmentMetadata = null; - Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix(MetadataFilenameUtils.METADATA_PREFIX); - Optional latestMetadataFile = metadataFiles.stream().max(METADATA_FILENAME_COMPARATOR); + List metadataFiles = remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + MetadataFilenameUtils.METADATA_PREFIX, + 1 + ); - if (latestMetadataFile.isPresent()) { - logger.info("Reading latest Metadata file {}", latestMetadataFile.get()); - remoteSegmentMetadata = readMetadataFile(latestMetadataFile.get()); + if (metadataFiles.isEmpty() == false) { + String latestMetadataFile = metadataFiles.get(0); + logger.info("Reading latest Metadata file {}", latestMetadataFile); + remoteSegmentMetadata = readMetadataFile(latestMetadataFile); } else { logger.info("No metadata file found, this can happen for new index with no data uploaded to remote segment store"); } @@ -187,8 +182,7 @@ private RemoteSegmentMetadata readMetadataFile(String metadataFilename) throws I try (IndexInput indexInput = remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)) { byte[] metadataBytes = new byte[(int) indexInput.length()]; indexInput.readBytes(metadataBytes, 0, (int) indexInput.length()); - RemoteSegmentMetadata metadata = metadataStreamWrapper.readStream(new ByteArrayIndexInput(metadataFilename, metadataBytes)); - return metadata; + return metadataStreamWrapper.readStream(new ByteArrayIndexInput(metadataFilename, metadataBytes)); } } @@ -242,56 +236,43 @@ static class MetadataFilenameUtils { public static final String SEPARATOR = "__"; public static final String METADATA_PREFIX = "metadata"; - /** - * Comparator to sort the metadata filenames. The order of sorting is: Primary Term, Generation, UUID - * Even though UUID sort does not provide any info on recency, it provides a consistent way to sort the filenames. - */ - static class MetadataFilenameComparator implements Comparator { - @Override - public int compare(String first, String second) { - String[] firstTokens = first.split(SEPARATOR); - String[] secondTokens = second.split(SEPARATOR); - if (!firstTokens[0].equals(secondTokens[0])) { - return firstTokens[0].compareTo(secondTokens[0]); - } - long firstPrimaryTerm = getPrimaryTerm(firstTokens); - long secondPrimaryTerm = getPrimaryTerm(secondTokens); - if (firstPrimaryTerm != secondPrimaryTerm) { - return firstPrimaryTerm > secondPrimaryTerm ? 1 : -1; - } else { - long firstGeneration = getGeneration(firstTokens); - long secondGeneration = getGeneration(secondTokens); - if (firstGeneration != secondGeneration) { - return firstGeneration > secondGeneration ? 1 : -1; - } else { - return getUuid(firstTokens).compareTo(getUuid(secondTokens)); - } - } - } - } - static String getMetadataFilePrefixForCommit(long primaryTerm, long generation) { - return String.join(SEPARATOR, METADATA_PREFIX, Long.toString(primaryTerm), Long.toString(generation, Character.MAX_RADIX)); + return String.join( + SEPARATOR, + METADATA_PREFIX, + RemoteStoreUtils.invertLong(primaryTerm), + RemoteStoreUtils.invertLong(generation) + ); } // Visible for testing - static String getMetadataFilename(long primaryTerm, long generation, String uuid) { - return String.join(SEPARATOR, getMetadataFilePrefixForCommit(primaryTerm, generation), uuid); + static String getMetadataFilename( + long primaryTerm, + long generation, + long translogGeneration, + long uploadCounter, + int metadataVersion + ) { + return String.join( + SEPARATOR, + METADATA_PREFIX, + RemoteStoreUtils.invertLong(primaryTerm), + RemoteStoreUtils.invertLong(generation), + RemoteStoreUtils.invertLong(translogGeneration), + RemoteStoreUtils.invertLong(uploadCounter), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(metadataVersion) + ); } // Visible for testing static long getPrimaryTerm(String[] filenameTokens) { - return Long.parseLong(filenameTokens[1]); + return RemoteStoreUtils.invertLong(filenameTokens[1]); } // Visible for testing static long getGeneration(String[] filenameTokens) { - return Long.parseLong(filenameTokens[2], Character.MAX_RADIX); - } - - // Visible for testing - static String getUuid(String[] filenameTokens) { - return filenameTokens[3]; + return RemoteStoreUtils.invertLong(filenameTokens[2]); } } @@ -379,7 +360,6 @@ public IndexInput openInput(String name, IOContext context) throws IOException { @Override public void acquireLock(long primaryTerm, long generation, String acquirerId) throws IOException { String metadataFile = getMetadataFileForCommit(primaryTerm, generation); - mdLockManager.acquire(FileLockInfo.getLockInfoBuilder().withFileToLock(metadataFile).withAcquirerId(acquirerId).build()); } @@ -408,13 +388,19 @@ public void releaseLock(long primaryTerm, long generation, String acquirerId) th @Override public Boolean isLockAcquired(long primaryTerm, long generation) throws IOException { String metadataFile = getMetadataFileForCommit(primaryTerm, generation); + return isLockAcquired(metadataFile); + } + + // Visible for testing + Boolean isLockAcquired(String metadataFile) throws IOException { return mdLockManager.isAcquired(FileLockInfo.getLockInfoBuilder().withFileToLock(metadataFile).build()); } // Visible for testing String getMetadataFileForCommit(long primaryTerm, long generation) throws IOException { - Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix( - MetadataFilenameUtils.getMetadataFilePrefixForCommit(primaryTerm, generation) + List metadataFiles = remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + MetadataFilenameUtils.getMetadataFilePrefixForCommit(primaryTerm, generation), + 1 ); if (metadataFiles.isEmpty()) { @@ -432,33 +418,24 @@ String getMetadataFileForCommit(long primaryTerm, long generation) throws IOExce + metadataFiles.size() ); } - return metadataFiles.iterator().next(); + return metadataFiles.get(0); } - public void copyFrom(Directory from, String src, String dest, IOContext context, boolean useCommonSuffix, String checksum) - throws IOException { + public void copyFrom(Directory from, String src, String dest, IOContext context, String checksum) throws IOException { String remoteFilename; - if (useCommonSuffix) { - remoteFilename = dest + SEGMENT_NAME_UUID_SEPARATOR + this.commonFilenameSuffix; - } else { - remoteFilename = getNewRemoteSegmentFilename(dest); - } + remoteFilename = getNewRemoteSegmentFilename(dest); remoteDataDirectory.copyFrom(from, src, remoteFilename, context); UploadedSegmentMetadata segmentMetadata = new UploadedSegmentMetadata(src, remoteFilename, checksum, from.fileLength(src)); segmentsUploadedToRemoteStore.put(src, segmentMetadata); } - public void copyFrom(Directory from, String src, String dest, IOContext context, boolean useCommonSuffix) throws IOException { - copyFrom(from, src, dest, context, useCommonSuffix, getChecksumOfLocalFile(from, src)); - } - /** * Copies an existing src file from directory from to a non-existent file dest in this directory. * Once the segment is uploaded to remote segment store, update the cache accordingly. */ @Override public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { - copyFrom(from, src, dest, context, false); + copyFrom(from, src, dest, context, getChecksumOfLocalFile(from, src)); } /** @@ -486,13 +463,16 @@ public void uploadMetadata( Collection segmentFiles, SegmentInfos segmentInfosSnapshot, Directory storeDirectory, - long primaryTerm + long primaryTerm, + long translogGeneration ) throws IOException { synchronized (this) { String metadataFilename = MetadataFilenameUtils.getMetadataFilename( primaryTerm, segmentInfosSnapshot.getGeneration(), - this.commonFilenameSuffix + translogGeneration, + metadataUploadCounter.incrementAndGet(), + RemoteSegmentMetadata.CURRENT_VERSION ); try { IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT); @@ -569,15 +549,6 @@ public Map getSegmentsUploadedToRemoteStore() { return Collections.unmodifiableMap(this.segmentsUploadedToRemoteStore); } - public Map getSegmentsUploadedToRemoteStore(long primaryTerm, long generation) throws IOException { - String metadataFile = getMetadataFileForCommit(primaryTerm, generation); - - Map segmentsUploadedToRemoteStore = new ConcurrentHashMap<>( - readMetadataFile(metadataFile).getMetadata() - ); - return Collections.unmodifiableMap(segmentsUploadedToRemoteStore); - } - /** * Delete stale segment and metadata files * One metadata file is kept per commit (refresh updates the same file). To read segments uploaded to remote store, @@ -585,9 +556,11 @@ public Map getSegmentsUploadedToRemoteStore(lon * @param lastNMetadataFilesToKeep number of metadata files to keep * @throws IOException in case of I/O error while reading from / writing to remote segment store */ - private void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException { - Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix(MetadataFilenameUtils.METADATA_PREFIX); - List sortedMetadataFileList = metadataFiles.stream().sorted(METADATA_FILENAME_COMPARATOR).collect(Collectors.toList()); + public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException { + List sortedMetadataFileList = remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + MetadataFilenameUtils.METADATA_PREFIX, + Integer.MAX_VALUE + ); if (sortedMetadataFileList.size() <= lastNMetadataFilesToKeep) { logger.info( "Number of commits in remote segment store={}, lastNMetadataFilesToKeep={}", @@ -598,21 +571,12 @@ private void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOExceptio } List metadataFilesEligibleToDelete = sortedMetadataFileList.subList( - 0, - sortedMetadataFileList.size() - lastNMetadataFilesToKeep + lastNMetadataFilesToKeep, + sortedMetadataFileList.size() ); List metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream().filter(metadataFile -> { try { - // TODO: add snapshot interop feature flag here as that will be the first feature to use lock - // manager. - boolean lockManagerEnabled = false; - if (!lockManagerEnabled) { - return true; - } - return !isLockAcquired( - MetadataFilenameUtils.getPrimaryTerm(metadataFile.split(MetadataFilenameUtils.SEPARATOR)), - MetadataFilenameUtils.getGeneration(metadataFile.split(MetadataFilenameUtils.SEPARATOR)) - ); + return !isLockAcquired(metadataFile); } catch (IOException e) { logger.error( "skipping metadata file (" @@ -699,7 +663,10 @@ public void deleteStaleSegmentsAsync(int lastNMetadataFilesToKeep) { Return true if it deleted it successfully */ private boolean deleteIfEmpty() throws IOException { - Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix(MetadataFilenameUtils.METADATA_PREFIX); + Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + MetadataFilenameUtils.METADATA_PREFIX, + 1 + ); if (metadataFiles.size() != 0) { logger.info("Remote directory still has files , not deleting the path"); return false; diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java index 7eaab67ddb5a5..9b3240823f368 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java @@ -300,6 +300,11 @@ public void onDelete() { translog.onDelete(); } + @Override + public Translog.TranslogGeneration getTranslogGeneration() { + return translog.getGeneration(); + } + /** * Reads operations from the translog * @param location location of translog diff --git a/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java index 58ee8c0fd39e7..dd5593b6d79cd 100644 --- a/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java @@ -122,4 +122,9 @@ public Translog.Snapshot newChangesSnapshot(long fromSeqNo, long toSeqNo, boolea } public void onDelete() {} + + @Override + public Translog.TranslogGeneration getTranslogGeneration() { + return null; + } } diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogManager.java b/server/src/main/java/org/opensearch/index/translog/TranslogManager.java index 420d6cdc43bbf..78aaa1bc13a00 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogManager.java @@ -131,4 +131,6 @@ public interface TranslogManager { Clean up if any needed on deletion of index */ void onDelete(); + + Translog.TranslogGeneration getTranslogGeneration(); } diff --git a/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java index f139a5d4e3bb1..4a2eeabeb7e58 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java @@ -131,7 +131,7 @@ private void testListBlobsByPrefixInSortedOrder(int limit, BlobContainer.BlobNam List blobsInFileSystem = new ArrayList<>(); for (int i = 0; i < 10; i++) { - final String blobName = randomAlphaOfLengthBetween(1, 20).toLowerCase(Locale.ROOT); + final String blobName = randomAlphaOfLengthBetween(10, 20).toLowerCase(Locale.ROOT); final byte[] blobData = randomByteArrayOfLength(randomIntBetween(1, frequently() ? 512 : 1 << 20)); // rarely up to 1mb Files.write(path.resolve(blobName), blobData); blobsInFileSystem.add(blobName); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java index 15f1585bd1477..8ee5fcf0da9d7 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java @@ -12,6 +12,8 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.junit.Before; +import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.support.PlainBlobMetadata; @@ -23,15 +25,19 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.mockito.Mockito.mock; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.doAnswer; public class RemoteDirectoryTests extends OpenSearchTestCase { private BlobContainer blobContainer; @@ -146,6 +152,54 @@ public void testFileLengthIOException() throws IOException { assertThrows(IOException.class, () -> remoteDirectory.fileLength("segment_1")); } + public void testListFilesByPrefixInLexicographicOrder() throws IOException { + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + latchedActionListener.onResponse(List.of(new PlainBlobMetadata("metadata_1", 1))); + return null; + }).when(blobContainer) + .listBlobsByPrefixInSortedOrder( + eq("metadata"), + eq(1), + eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), + any(ActionListener.class) + ); + + assertEquals(List.of("metadata_1"), remoteDirectory.listFilesByPrefixInLexicographicOrder("metadata", 1)); + } + + public void testListFilesByPrefixInLexicographicOrderEmpty() throws IOException { + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + latchedActionListener.onResponse(List.of()); + return null; + }).when(blobContainer) + .listBlobsByPrefixInSortedOrder( + eq("metadata"), + eq(1), + eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), + any(ActionListener.class) + ); + + assertEquals(List.of(), remoteDirectory.listFilesByPrefixInLexicographicOrder("metadata", 1)); + } + + public void testListFilesByPrefixInLexicographicOrderException() { + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + latchedActionListener.onFailure(new IOException("Error")); + return null; + }).when(blobContainer) + .listBlobsByPrefixInSortedOrder( + eq("metadata"), + eq(1), + eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), + any(ActionListener.class) + ); + + assertThrows(IOException.class, () -> remoteDirectory.listFilesByPrefixInLexicographicOrder("metadata", 1)); + } + public void testGetPendingDeletions() { assertThrows(UnsupportedOperationException.class, () -> remoteDirectory.getPendingDeletions()); } @@ -165,5 +219,4 @@ public void testRename() { public void testObtainLock() { assertThrows(UnsupportedOperationException.class, () -> remoteDirectory.obtainLock("segment_1")); } - } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index 324315505987b..bf4b2a14f2567 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -11,8 +11,11 @@ import org.apache.lucene.store.Directory; import org.junit.Before; import org.mockito.ArgumentCaptor; +import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.Settings; @@ -28,15 +31,16 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.Collections; import java.util.List; import java.util.function.Supplier; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.doAnswer; public class RemoteSegmentStoreDirectoryFactoryTests extends OpenSearchTestCase { @@ -68,7 +72,12 @@ public void testNewDirectory() throws IOException { when(repository.blobStore()).thenReturn(blobStore); when(repository.basePath()).thenReturn(new BlobPath().add("base_path")); when(blobStore.blobContainer(any())).thenReturn(blobContainer); - when(blobContainer.listBlobs()).thenReturn(Collections.emptyMap()); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + latchedActionListener.onResponse(List.of()); + return null; + }).when(blobContainer) + .listBlobsByPrefixInSortedOrder(any(), eq(1), eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), any(ActionListener.class)); when(repositoriesService.repository("remote_store_repository")).thenReturn(repository); @@ -81,7 +90,12 @@ public void testNewDirectory() throws IOException { assertEquals("base_path/uuid_1/0/segments/metadata/", blobPaths.get(1).buildAsString()); assertEquals("base_path/uuid_1/0/segments/lock_files/", blobPaths.get(2).buildAsString()); - verify(blobContainer).listBlobsByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX); + verify(blobContainer).listBlobsByPrefixInSortedOrder( + eq(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX), + eq(1), + eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), + any() + ); verify(repositoriesService, times(2)).repository("remote_store_repository"); } } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 66e4b9a357b85..c37893877253e 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -32,6 +32,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; @@ -71,6 +72,10 @@ public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase { private SegmentInfos segmentInfos; private ThreadPool threadPool; + private final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 23, 34, 1, 1); + private final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 13, 34, 1, 1); + private final String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(10, 38, 34, 1, 1); + @Before public void setup() throws IOException { remoteDataDirectory = mock(RemoteDirectory.class); @@ -119,50 +124,16 @@ public void testUploadedSegmentMetadataFromString() { assertEquals("_0.cfe::_0.cfe__uuidxyz::4567::372000", metadata.toString()); } - public void testGetMetadataFilename() { - // Generation 23 is replaced by n due to radix 32 - assertEquals( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX + "__12__n__uuid1", - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 23, "uuid1") - ); - } - public void testGetPrimaryTermGenerationUuid() { - String[] filenameTokens = "abc__12__n__uuid_xyz".split(RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR); + String[] filenameTokens = "abc__9223372036854775795__9223372036854775784__uuid_xyz".split( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR + ); assertEquals(12, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getPrimaryTerm(filenameTokens)); assertEquals(23, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getGeneration(filenameTokens)); - assertEquals("uuid_xyz", RemoteSegmentStoreDirectory.MetadataFilenameUtils.getUuid(filenameTokens)); - } - - public void testMetadataFilenameComparator() { - List metadataFilenames = new ArrayList<>( - List.of( - "abc__10__20__uuid1", - "abc__12__2__uuid2", - "pqr__1__1__uuid0", - "abc__3__n__uuid3", - "abc__10__8__uuid8", - "abc__3__a__uuid4", - "abc__3__a__uuid5" - ) - ); - metadataFilenames.sort(RemoteSegmentStoreDirectory.METADATA_FILENAME_COMPARATOR); - assertEquals( - List.of( - "abc__3__a__uuid4", - "abc__3__a__uuid5", - "abc__3__n__uuid3", - "abc__10__8__uuid8", - "abc__10__20__uuid1", - "abc__12__2__uuid2", - "pqr__1__1__uuid0" - ), - metadataFilenames - ); } public void testInitException() throws IOException { - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenThrow( + when(remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, 1)).thenThrow( new IOException("Error") ); @@ -262,29 +233,42 @@ private ByteArrayIndexInput createMetadataFileBytes(Map segmentF } private Map> populateMetadata() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = new ArrayList<>(); + + metadataFiles.add(metadataFilename); + metadataFiles.add(metadataFilename2); + metadataFiles.add(metadataFilename3); + + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(List.of(metadataFilename)); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + Integer.MAX_VALUE + ) + ).thenReturn(metadataFiles); Map> metadataFilenameContentMapping = Map.of( - "metadata__1__5__abc", + metadataFilename, getDummyMetadata("_0", 1), - "metadata__1__6__pqr", + metadataFilename2, getDummyMetadata("_0", 1), - "metadata__2__1__zxv", + metadataFilename3, getDummyMetadata("_0", 1) ); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc"), 1, 5) + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenAnswer( + I -> createMetadataFileBytes(metadataFilenameContentMapping.get(metadataFilename), 23, 12) ); - when(remoteMetadataDirectory.openInput("metadata__1__6__pqr", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__6__pqr"), 1, 6) + when(remoteMetadataDirectory.openInput(metadataFilename2, IOContext.DEFAULT)).thenAnswer( + I -> createMetadataFileBytes(metadataFilenameContentMapping.get(metadataFilename2), 13, 12) ); - when(remoteMetadataDirectory.openInput("metadata__2__1__zxv", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv"), 1, 2), - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv"), 1, 2) + when(remoteMetadataDirectory.openInput(metadataFilename3, IOContext.DEFAULT)).thenAnswer( + I -> createMetadataFileBytes(metadataFilenameContentMapping.get(metadataFilename3), 38, 10) ); return metadataFilenameContentMapping; @@ -293,9 +277,12 @@ private Map> populateMetadata() throws IOException { public void testInit() throws IOException { populateMetadata(); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv") - ); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(List.of(metadataFilename)); remoteSegmentStoreDirectory.init(); @@ -399,15 +386,15 @@ public void testOpenInputException() throws IOException { public void testAcquireLock() throws IOException { populateMetadata(); remoteSegmentStoreDirectory.init(); - String mdFile = "xyz"; String acquirerId = "test-acquirer"; long testPrimaryTerm = 1; long testGeneration = 5; List metadataFiles = List.of("metadata__1__5__abc"); when( - remoteMetadataDirectory.listFilesByPrefix( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration) + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration), + 1 ) ).thenReturn(metadataFiles); @@ -437,8 +424,9 @@ public void testReleaseLock() throws IOException { List metadataFiles = List.of("metadata__1__5__abc"); when( - remoteMetadataDirectory.listFilesByPrefix( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration) + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration), + 1 ) ).thenReturn(metadataFiles); @@ -454,8 +442,9 @@ public void testIsAcquired() throws IOException { List metadataFiles = List.of("metadata__1__5__abc"); when( - remoteMetadataDirectory.listFilesByPrefix( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration) + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration), + 1 ) ).thenReturn(metadataFiles); @@ -471,8 +460,9 @@ public void testIsAcquiredException() throws IOException { List metadataFiles = new ArrayList<>(); when( - remoteMetadataDirectory.listFilesByPrefix( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration) + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration), + 1 ) ).thenReturn(metadataFiles); @@ -482,14 +472,10 @@ public void testIsAcquiredException() throws IOException { public void testGetMetadataFileForCommit() throws IOException { long testPrimaryTerm = 2; long testGeneration = 3; - List metadataFiles = List.of( - "metadata__1__5__abc", - "metadata__" + testPrimaryTerm + "__" + testGeneration + "__pqr", - "metadata__2__1__zxv" - ); when( - remoteMetadataDirectory.listFilesByPrefix( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration) + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration), + 1 ) ).thenReturn(List.of("metadata__" + testPrimaryTerm + "__" + testGeneration + "__pqr")); @@ -498,33 +484,6 @@ public void testGetMetadataFileForCommit() throws IOException { } - public void testGetSegmentsUploadedToRemoteStore() throws IOException { - long testPrimaryTerm = 1; - long testGeneration = 5; - - List metadataFiles = List.of("metadata__1__5__abc"); - when( - remoteMetadataDirectory.listFilesByPrefix( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilePrefixForCommit(testPrimaryTerm, testGeneration) - ) - ).thenReturn(metadataFiles); - - Map> metadataFilenameContentMapping = Map.of( - "metadata__1__5__abc", - getDummyMetadata("_0", 5), - "metadata__1__6__pqr", - getDummyMetadata("_0", 6), - "metadata__2__1__zxv", - getDummyMetadata("_0", 1) - ); - - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc"), 1, 5) - ); - - assert (remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore(testPrimaryTerm, testGeneration).containsKey("segments_5")); - } - public void testCopyFrom() throws IOException { String filename = "_100.si"; populateMetadata(); @@ -556,46 +515,20 @@ public void testCopyFromException() throws IOException { storeDirectory.close(); } - public void testCopyFromOverride() throws IOException { - String filename = "_100.si"; - populateMetadata(); - remoteSegmentStoreDirectory.init(); - - Directory storeDirectory = LuceneTestCase.newDirectory(); - IndexOutput indexOutput = storeDirectory.createOutput(filename, IOContext.DEFAULT); - indexOutput.writeString("Hello World!"); - CodecUtil.writeFooter(indexOutput); - indexOutput.close(); - storeDirectory.sync(List.of(filename)); - - assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); - remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT, true); - RemoteSegmentStoreDirectory.UploadedSegmentMetadata uploadedSegmentMetadata = remoteSegmentStoreDirectory - .getSegmentsUploadedToRemoteStore() - .get(filename); - assertNotNull(uploadedSegmentMetadata); - remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT, true); - assertEquals( - uploadedSegmentMetadata.toString(), - remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().get(filename).toString() - ); - - storeDirectory.close(); - } - public void testContainsFile() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = List.of(metadataFilename); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512"); metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024"); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( - createMetadataFileBytes(metadata, 1, 5) - ); + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(createMetadataFileBytes(metadata, 1, 5)); remoteSegmentStoreDirectory.init(); @@ -625,7 +558,7 @@ public void testUploadMetadataEmpty() throws IOException { Collection segmentFiles = List.of("s1", "s2", "s3"); assertThrows( NoSuchFileException.class, - () -> remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L) + () -> remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L, 34L) ); } @@ -637,16 +570,19 @@ public void testUploadMetadataNonEmpty() throws IOException { BytesStreamOutput output = new BytesStreamOutput(); IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); - long generation = segmentInfos.getGeneration(); - when(storeDirectory.createOutput(startsWith("metadata__12__" + generation), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + String generation = RemoteStoreUtils.invertLong(segmentInfos.getGeneration()); + String primaryTerm = RemoteStoreUtils.invertLong(12); + when(storeDirectory.createOutput(startsWith("metadata__" + primaryTerm + "__" + generation), eq(IOContext.DEFAULT))).thenReturn( + indexOutput + ); Collection segmentFiles = List.of("_0.si", "_0.cfe", "_0.cfs", "segments_1"); - remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L); + remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L, 34L); verify(remoteMetadataDirectory).copyFrom( eq(storeDirectory), - startsWith("metadata__12__" + generation), - startsWith("metadata__12__" + generation), + startsWith("metadata__" + primaryTerm + "__" + generation), + startsWith("metadata__" + primaryTerm + "__" + generation), eq(IOContext.DEFAULT) ); @@ -669,10 +605,13 @@ public void testUploadMetadataNonEmpty() throws IOException { } public void testNoMetadataHeaderCorruptIndexException() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = List.of(metadataFilename); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); @@ -683,16 +622,19 @@ public void testNoMetadataHeaderCorruptIndexException() throws IOException { indexOutput.writeMapOfStrings(metadata); indexOutput.close(); ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } public void testInvalidCodecHeaderCorruptIndexException() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = List.of(metadataFilename); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); @@ -705,16 +647,19 @@ public void testInvalidCodecHeaderCorruptIndexException() throws IOException { CodecUtil.writeFooter(indexOutput); indexOutput.close(); ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } public void testHeaderMinVersionCorruptIndexException() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = List.of(metadataFilename); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); @@ -727,16 +672,19 @@ public void testHeaderMinVersionCorruptIndexException() throws IOException { CodecUtil.writeFooter(indexOutput); indexOutput.close(); ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); assertThrows(IndexFormatTooOldException.class, () -> remoteSegmentStoreDirectory.init()); } public void testHeaderMaxVersionCorruptIndexException() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = List.of(metadataFilename); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); @@ -749,16 +697,19 @@ public void testHeaderMaxVersionCorruptIndexException() throws IOException { CodecUtil.writeFooter(indexOutput); indexOutput.close(); ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); assertThrows(IndexFormatTooNewException.class, () -> remoteSegmentStoreDirectory.init()); } public void testIncorrectChecksumCorruptIndexException() throws IOException { - List metadataFiles = List.of("metadata__1__5__abc"); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( - metadataFiles - ); + List metadataFiles = List.of(metadataFilename); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512"); @@ -775,16 +726,19 @@ public void testIncorrectChecksumCorruptIndexException() throws IOException { indexOutputSpy.close(); ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } public void testDeleteStaleCommitsException() throws Exception { populateMetadata(); - when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenThrow( - new IOException("Error reading") - ); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + Integer.MAX_VALUE + ) + ).thenThrow(new IOException("Error reading")); // popluateMetadata() adds stub to return 3 metadata files // We are passing lastNMetadataFilesToKeep=2 here to validate that in case of exception deleteFile is not @@ -840,20 +794,20 @@ public void testDeleteStaleCommitsActualDelete() throws Exception { // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); - for (String metadata : metadataFilenameContentMapping.get("metadata__1__5__abc").values()) { + for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; verify(remoteDataDirectory).deleteFile(uploadedFilename); } ; assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); - verify(remoteMetadataDirectory).deleteFile("metadata__1__5__abc"); + verify(remoteMetadataDirectory).deleteFile(metadataFilename3); } public void testDeleteStaleCommitsActualDeleteIOException() throws Exception { Map> metadataFilenameContentMapping = populateMetadata(); remoteSegmentStoreDirectory.init(); - String segmentFileWithException = metadataFilenameContentMapping.get("metadata__1__5__abc") + String segmentFileWithException = metadataFilenameContentMapping.get(metadataFilename3) .values() .stream() .findAny() @@ -864,20 +818,19 @@ public void testDeleteStaleCommitsActualDeleteIOException() throws Exception { // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); - for (String metadata : metadataFilenameContentMapping.get("metadata__1__5__abc").values()) { + for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; verify(remoteDataDirectory).deleteFile(uploadedFilename); } - ; assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); - verify(remoteMetadataDirectory, times(0)).deleteFile("metadata__1__5__abc"); + verify(remoteMetadataDirectory, times(0)).deleteFile(metadataFilename3); } public void testDeleteStaleCommitsActualDeleteNoSuchFileException() throws Exception { Map> metadataFilenameContentMapping = populateMetadata(); remoteSegmentStoreDirectory.init(); - String segmentFileWithException = metadataFilenameContentMapping.get("metadata__1__5__abc") + String segmentFileWithException = metadataFilenameContentMapping.get(metadataFilename) .values() .stream() .findAny() @@ -888,13 +841,12 @@ public void testDeleteStaleCommitsActualDeleteNoSuchFileException() throws Excep // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); - for (String metadata : metadataFilenameContentMapping.get("metadata__1__5__abc").values()) { + for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; verify(remoteDataDirectory).deleteFile(uploadedFilename); } - ; assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); - verify(remoteMetadataDirectory).deleteFile("metadata__1__5__abc"); + verify(remoteMetadataDirectory).deleteFile(metadataFilename3); } public void testSegmentMetadataCurrentVersion() { @@ -909,6 +861,20 @@ public void testSegmentMetadataCurrentVersion() { assertEquals(RemoteSegmentMetadata.CURRENT_VERSION, 1); } + public void testMetadataFileNameOrder() { + String file1 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 21, 23, 1, 1); + String file2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 38, 1, 1); + String file3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(18, 12, 26, 1, 1); + String file4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 10, 1); + String file5 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 1, 1); + String file6 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 5, 1); + + List actualList = new ArrayList<>(List.of(file1, file2, file3, file4, file5, file6)); + actualList.sort(String::compareTo); + + assertEquals(List.of(file3, file2, file4, file6, file5, file1), actualList); + } + private static class WrapperIndexOutput extends IndexOutput { public IndexOutput indexOutput; From bcaf49455fc6dacd1e19e32b8457a8bc33991c7a Mon Sep 17 00:00:00 2001 From: Ankit Kala Date: Thu, 6 Jul 2023 21:53:17 +0530 Subject: [PATCH 058/186] Flaky test: Add condition to ensure data correctness on primary promotion (#8431) Signed-off-by: Ankit Kala --- .../org/opensearch/indices/replication/SegmentReplicationIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 2b879f1c37d88..ac57c78d20b73 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -733,6 +733,7 @@ public void testDropPrimaryDuringReplication() throws Exception { // start another replica. dataNodes.add(internalCluster().startDataOnlyNode()); ensureGreen(INDEX_NAME); + waitForSearchableDocs(initialDocCount, dataNodes); // index another doc and refresh - without this the new replica won't catch up. String docId = String.valueOf(initialDocCount + 1); From bea26f97c66070581070764fe4790d2f0579c8e7 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Thu, 6 Jul 2023 11:24:03 -0700 Subject: [PATCH 059/186] [Segment Replication] Remove Restriction of strictly using only docrep for system indices and hidden indices (#8200) * remove logic of strictly using only docrep for system indices and hidden indices. Signed-off-by: Rishikesh1159 * Remove segrep-system indices related tests. Signed-off-by: Rishikesh1159 * Fix failing tests. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 Signed-off-by: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> --- .../SegmentReplicationClusterSettingIT.java | 66 --------- .../remotestore/CreateRemoteIndexIT.java | 52 ------- .../metadata/MetadataCreateIndexService.java | 28 +--- .../MetadataCreateIndexServiceTests.java | 130 +++--------------- 4 files changed, 25 insertions(+), 251 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java index 57578cdbfa8e8..c1326c1b50c9e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java @@ -15,16 +15,8 @@ import org.opensearch.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.transport.MockTransportService; - -import java.util.Collection; -import java.util.Collections; -import java.util.Arrays; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; @@ -60,40 +52,6 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } - public static class TestPlugin extends Plugin implements SystemIndexPlugin { - @Override - public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList( - new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System index for [" + getTestClass().getName() + ']') - ); - } - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(SegmentReplicationClusterSettingIT.TestPlugin.class, MockTransportService.TestPlugin.class); - } - - public void testSystemIndexWithSegmentReplicationClusterSetting() throws Exception { - - // Starting two nodes with primary and replica shards respectively. - final String primaryNode = internalCluster().startNode(); - createIndex(SYSTEM_INDEX_NAME); - ensureYellowAndNoInitializingShards(SYSTEM_INDEX_NAME); - final String replicaNode = internalCluster().startNode(); - ensureGreen(SYSTEM_INDEX_NAME); - final GetSettingsResponse response = client().admin() - .indices() - .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) - .actionGet(); - assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); - - // Verify index setting isSegRepEnabled is false. - Index index = resolveIndex(SYSTEM_INDEX_NAME); - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); - assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); - } - public void testIndexReplicationSettingOverridesSegRepClusterSetting() throws Exception { Settings settings = Settings.builder().put(CLUSTER_SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build(); final String ANOTHER_INDEX = "test-index"; @@ -165,28 +123,4 @@ public void testIndexReplicationSettingOverridesDocRepClusterSetting() throws Ex assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), false); } - public void testHiddenIndicesWithReplicationStrategyClusterSetting() throws Exception { - final String primaryNode = internalCluster().startNode(); - final String replicaNode = internalCluster().startNode(); - prepareCreate( - INDEX_NAME, - Settings.builder() - // we want to set index as hidden - .put("index.hidden", true) - ).get(); - ensureGreen(INDEX_NAME); - - // Verify that document replication strategy is used for hidden indices. - final GetSettingsResponse response = client().admin() - .indices() - .getSettings(new GetSettingsRequest().indices(INDEX_NAME).includeDefaults(true)) - .actionGet(); - assertEquals(response.getSetting(INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); - - // Verify index setting isSegRepEnabled. - Index index = resolveIndex(INDEX_NAME); - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); - assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); - } - } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index 46966e289e75e..fda344acad166 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -12,24 +12,15 @@ import org.junit.Before; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; -import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.index.IndexSettings; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; - import static org.hamcrest.Matchers.containsString; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; @@ -68,20 +59,6 @@ protected Settings nodeSettings(int nodeOriginal) { return builder.build(); } - public static class TestPlugin extends Plugin implements SystemIndexPlugin { - @Override - public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList( - new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System index for [" + getTestClass().getName() + ']') - ); - } - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(CreateRemoteIndexIT.TestPlugin.class); - } - @Override protected Settings featureFlagSettings() { return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); @@ -130,35 +107,6 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception { ); } - private static final String SYSTEM_INDEX_NAME = ".test-system-index"; - - public void testSystemIndexWithRemoteStoreClusterSetting() throws Exception { - createIndex(SYSTEM_INDEX_NAME); - ensureGreen(SYSTEM_INDEX_NAME); - final GetSettingsResponse response = client().admin() - .indices() - .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) - .actionGet(); - // Verify that Document replication strategy is used - assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); - assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REMOTE_STORE_ENABLED), "false"); - } - - public void testSystemIndexWithRemoteStoreIndexSettings() throws Exception { - prepareCreate( - SYSTEM_INDEX_NAME, - Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).put(SETTING_REMOTE_STORE_ENABLED, true) - ).get(); - ensureGreen(SYSTEM_INDEX_NAME); - final GetSettingsResponse response = client().admin() - .indices() - .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) - .actionGet(); - // Verify that Document replication strategy is used - assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); - assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REMOTE_STORE_ENABLED), "false"); - } - public void testRemoteStoreDisabledByUser() throws Exception { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 0a49feec61621..3fff6e8823e9e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -72,7 +72,6 @@ import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -137,7 +136,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; -import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; @@ -585,8 +583,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - systemIndices.validateSystemIndex(request.index()) + indexSettingProviders ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -650,8 +647,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - systemIndices.validateSystemIndex(request.index()) + indexSettingProviders ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -731,8 +727,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - sourceMetadata.isSystem() + indexSettingProviders ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -815,8 +810,7 @@ static Settings aggregateIndexSettings( Settings settings, IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator, - Set indexSettingProviders, - boolean isSystemIndex + Set indexSettingProviders ) { // Create builders for the template and request settings. We transform these into builders // because we may want settings to be "removed" from these prior to being set on the new @@ -900,18 +894,8 @@ static Settings aggregateIndexSettings( indexSettingsBuilder.put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName()); indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - if (isSystemIndex || IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings())) { - logger.warn( - "Setting replication.type: DOCUMENT will be used for Index until Segment Replication supports System and Hidden indices" - ); - indexSettingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); - if (FeatureFlags.isEnabled(REMOTE_STORE)) { - indexSettingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, false); - } - } else { - updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings); - updateRemoteStoreSettings(indexSettingsBuilder, request.settings(), settings); - } + updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings); + updateRemoteStoreSettings(indexSettingsBuilder, request.settings(), settings); if (sourceMetadata != null) { assert request.resizeType() != null; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 94a1ca9992cb6..c70fedbaf24bd 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -817,8 +817,7 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); @@ -880,8 +879,7 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); @@ -903,8 +901,7 @@ public void testDefaultSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); @@ -919,8 +916,7 @@ public void testSettingsFromClusterState() { Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); @@ -957,8 +953,7 @@ public void testTemplateOrder() throws Exception { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); List resolvedAliases = resolveAndValidateAliases( request.index(), @@ -997,8 +992,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); @@ -1220,8 +1214,7 @@ public void testRemoteStoreNoUserOverrideConflictingReplicationTypeIndexSettings settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ) ); assertThat( @@ -1252,8 +1245,7 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1285,8 +1277,7 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1320,8 +1311,7 @@ public void testRemoteStoreDisabledByUserIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1355,8 +1345,7 @@ public void testRemoteStoreTranslogDisabledByUserIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1393,8 +1382,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1428,8 +1416,7 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1463,8 +1450,7 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1546,8 +1532,7 @@ public void testSoftDeletesDisabledIsRejected() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); }); assertThat( @@ -1576,8 +1561,7 @@ public void testValidateTranslogRetentionSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertWarnings( "Translog retention settings [index.translog.retention.age] " @@ -1624,8 +1608,7 @@ public void testDeprecatedSimpleFSStoreSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertWarnings( "[simplefs] is deprecated and will be removed in 2.0. Use [niofs], which offers equal " @@ -1644,8 +1627,7 @@ public void testClusterReplicationSetting() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); } @@ -1665,86 +1647,12 @@ public void testIndexSettingOverridesClusterReplicationSetting() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), - false + Collections.emptySet() ); // Verify if index setting overrides cluster replication setting assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); } - public void testHiddenIndexUsesDocumentReplication() { - Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder(); - // Set index setting replication type as DOCUMENT - requestSettings.put("index.hidden", true); - request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet(), - false - ); - // Verify replication type is Document Replication - assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); - } - - public void testSystemIndexUsesDocumentReplication() { - Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder(); - request.settings(requestSettings.build()); - // set isSystemIndex parameter as true - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet(), - true - ); - // Verify replication type is Document Replication - assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); - } - - public void testRemoteStoreDisabledForSystemIndices() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder(); - request.settings(requestSettings.build()); - // set isSystemIndex parameter as true - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet(), - true - ); - // Verify that remote store is disabled. - assertEquals(indexSettings.get(SETTING_REMOTE_STORE_ENABLED), "false"); - assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); - } - private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder); From 431b2464324a76e60fb446567504eae846f7b120 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Thu, 6 Jul 2023 14:09:25 -0700 Subject: [PATCH 060/186] [Search pipelines] Pass "adhocness" flag to processor factories (#8164) * [Search pipelines] Pass "adhocness" flag to processor factories A named search pipeline may be created with a PUT request, while an "anonymous" or "ad hoc" search pipeline can be defined in the search request source. In the latter case, we don't want to create any "resource-heavy" processors, since they're potentially increasing the cost of every search request, whereas names pipeline processors get reused. This change passes a configuration flag to a processor factory if it's being called as part of an ad hoc pipeline. The factory can use that information to avoid allocating expensive resources (maybe by throwing an exception instead). Signed-off-by: Michael Froh * Pass pipeline creation source as enum Thanks to @dblock for the suggestion to pass the pipeline creation source in a way that accounts for possible future pipeline sources (and lets us distinguish between actual named pipeline creation and the validation create() that executes before we write a pipeline definition to cluster state). Signed-off-by: Michael Froh * Move PipelineSource into PipelineContext and explicitly pass to create Signed-off-by: Michael Froh * Fix formatting on merge conflict resolution Signed-off-by: Michael Froh --------- Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../common/FilterQueryRequestProcessor.java | 29 ++--- .../common/RenameFieldResponseProcessor.java | 3 +- .../common/ScriptRequestProcessor.java | 29 +++-- .../SearchPipelineCommonModulePlugin.java | 4 +- .../FilterQueryRequestProcessorTests.java | 14 +-- .../RenameFieldResponseProcessorTests.java | 7 +- .../search_pipeline/50_script_processor.yml | 2 + .../plugins/SearchPipelinePlugin.java | 88 +++++++++++++- .../search/pipeline/PipelineWithMetrics.java | 35 +++++- .../opensearch/search/pipeline/Processor.java | 112 ++++++------------ .../pipeline/SearchPipelineService.java | 11 +- .../pipeline/SearchPipelineServiceTests.java | 109 ++++++++++++----- 13 files changed, 281 insertions(+), 163 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 619bb0d58d8ae..ab2ebcd9f3562 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -152,6 +152,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963)) - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) - Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) +- [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) ### Deprecated diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java index 7deb8faa03af6..d8862aa59cede 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java @@ -13,12 +13,12 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.pipeline.Processor; import org.opensearch.search.pipeline.SearchRequestProcessor; @@ -89,8 +89,8 @@ public SearchRequest processRequest(SearchRequest request) throws Exception { } static class Factory implements Processor.Factory { + private static final String QUERY_KEY = "query"; private final NamedXContentRegistry namedXContentRegistry; - public static final ParseField QUERY_FIELD = new ParseField("query"); Factory(NamedXContentRegistry namedXContentRegistry) { this.namedXContentRegistry = namedXContentRegistry; @@ -101,30 +101,21 @@ public FilterQueryRequestProcessor create( Map> processorFactories, String tag, String description, - Map config + Map config, + PipelineContext pipelineContext ) throws Exception { + Map query = ConfigurationUtils.readOptionalMap(TYPE, tag, config, QUERY_KEY); + if (query == null) { + throw new IllegalArgumentException("Did not specify the " + QUERY_KEY + " property in processor of type " + TYPE); + } try ( - XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config); + XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(query); InputStream stream = BytesReference.bytes(builder).streamInput(); XContentParser parser = XContentType.JSON.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, stream) ) { - XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT; - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if (QUERY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - return new FilterQueryRequestProcessor(tag, description, parseInnerQueryBuilder(parser)); - } - } - } + return new FilterQueryRequestProcessor(tag, description, parseInnerQueryBuilder(parser)); } - throw new IllegalArgumentException( - "Did not specify the " + QUERY_FIELD.getPreferredName() + " property in processor of type " + TYPE - ); } } } diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java index 4c40dda5928f0..c8b3c06a71562 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java @@ -140,7 +140,8 @@ public RenameFieldResponseProcessor create( Map> processorFactories, String tag, String description, - Map config + Map config, + PipelineContext pipelineContext ) throws Exception { String oldField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); String newField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field"); diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java index 015411e0701a4..43ab3d4622d6b 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java @@ -29,7 +29,8 @@ import org.opensearch.search.pipeline.common.helpers.SearchRequestMap; import java.io.InputStream; -import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.opensearch.ingest.ConfigurationUtils.newConfigurationException; @@ -127,6 +128,8 @@ SearchScript getPrecompiledSearchScript() { * Factory class for creating {@link ScriptRequestProcessor}. */ public static final class Factory implements Processor.Factory { + private static final List SCRIPT_CONFIG_KEYS = List.of("id", "source", "inline", "lang", "params", "options"); + private final ScriptService scriptService; /** @@ -138,33 +141,29 @@ public Factory(ScriptService scriptService) { this.scriptService = scriptService; } - /** - * Creates a new instance of {@link ScriptRequestProcessor}. - * - * @param registry The registry of processor factories. - * @param processorTag The processor's tag. - * @param description The processor's description. - * @param config The configuration options for the processor. - * @return The created {@link ScriptRequestProcessor} instance. - * @throws Exception if an error occurs during the creation process. - */ @Override public ScriptRequestProcessor create( Map> registry, String processorTag, String description, - Map config + Map config, + PipelineContext pipelineContext ) throws Exception { + Map scriptConfig = new HashMap<>(); + for (String key : SCRIPT_CONFIG_KEYS) { + Object val = config.remove(key); + if (val != null) { + scriptConfig.put(key, val); + } + } try ( - XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config); + XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(scriptConfig); InputStream stream = BytesReference.bytes(builder).streamInput(); XContentParser parser = XContentType.JSON.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream) ) { Script script = Script.parse(parser); - Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove); - // verify script is able to be compiled before successfully creating processor. SearchScript searchScript = null; try { diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java index dc25de460fdba..49681b80fdead 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java @@ -33,7 +33,7 @@ public SearchPipelineCommonModulePlugin() {} * @return A map of processor factories, where the keys are the processor types and the values are the corresponding factory instances. */ @Override - public Map> getRequestProcessors(Processor.Parameters parameters) { + public Map> getRequestProcessors(Parameters parameters) { return Map.of( FilterQueryRequestProcessor.TYPE, new FilterQueryRequestProcessor.Factory(parameters.namedXContentRegistry), @@ -43,7 +43,7 @@ public Map> getRequestProcesso } @Override - public Map> getResponseProcessors(Processor.Parameters parameters) { + public Map> getResponseProcessors(Parameters parameters) { return Map.of(RenameFieldResponseProcessor.TYPE, new RenameFieldResponseProcessor.Factory()); } } diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java index 1f355ac97c801..ecf746af556a2 100644 --- a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java @@ -16,6 +16,7 @@ import org.opensearch.test.AbstractBuilderTestCase; import java.util.Collections; +import java.util.HashMap; import java.util.Map; public class FilterQueryRequestProcessorTests extends AbstractBuilderTestCase { @@ -37,15 +38,14 @@ public void testFilterQuery() throws Exception { public void testFactory() throws Exception { FilterQueryRequestProcessor.Factory factory = new FilterQueryRequestProcessor.Factory(this.xContentRegistry()); - FilterQueryRequestProcessor processor = factory.create( - Collections.emptyMap(), - null, - null, - Map.of("query", Map.of("term", Map.of("field", "value"))) - ); + Map configMap = new HashMap<>(Map.of("query", Map.of("term", Map.of("field", "value")))); + FilterQueryRequestProcessor processor = factory.create(Collections.emptyMap(), null, null, configMap, null); assertEquals(new TermQueryBuilder("field", "value"), processor.filterQuery); // Missing "query" parameter: - expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), null, null, Collections.emptyMap())); + expectThrows( + IllegalArgumentException.class, + () -> factory.create(Collections.emptyMap(), null, null, Collections.emptyMap(), null) + ); } } diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java index a2fc7f6acfa7c..7f3a2acfbdc08 100644 --- a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java @@ -115,12 +115,15 @@ public void testFactory() throws Exception { config.put("target_field", newField); RenameFieldResponseProcessor.Factory factory = new RenameFieldResponseProcessor.Factory(); - RenameFieldResponseProcessor processor = factory.create(Collections.emptyMap(), null, null, config); + RenameFieldResponseProcessor processor = factory.create(Collections.emptyMap(), null, null, config, null); assertEquals(processor.getType(), "rename_field"); assertEquals(processor.getOldField(), oldField); assertEquals(processor.getNewField(), newField); assertFalse(processor.isIgnoreMissing()); - expectThrows(OpenSearchParseException.class, () -> factory.create(Collections.emptyMap(), null, null, Collections.emptyMap())); + expectThrows( + OpenSearchParseException.class, + () -> factory.create(Collections.emptyMap(), null, null, Collections.emptyMap(), null) + ); } } diff --git a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml index 9d855e8a1861a..7a01e68acf75c 100644 --- a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml +++ b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml @@ -17,6 +17,7 @@ teardown: "request_processors": [ { "script" : { + "tag": "empty_script", "lang": "painless", "source" : "" } @@ -38,6 +39,7 @@ teardown: "request_processors": [ { "script" : { + "tag": "working", "lang" : "painless", "source" : "ctx._source['size'] += 10; ctx._source['from'] = ctx._source['from'] <= 0 ? ctx._source['from'] : ctx._source['from'] - 1 ; ctx._source['explain'] = !ctx._source['explain']; ctx._source['version'] = !ctx._source['version']; ctx._source['seq_no_primary_term'] = !ctx._source['seq_no_primary_term']; ctx._source['track_scores'] = !ctx._source['track_scores']; ctx._source['track_total_hits'] = 1; ctx._source['min_score'] -= 0.9; ctx._source['terminate_after'] += 2; ctx._source['profile'] = !ctx._source['profile'];" } diff --git a/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java index 3d76bab93a60c..d2ef2b65c5944 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java @@ -8,13 +8,24 @@ package org.opensearch.plugins; +import org.opensearch.client.Client; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.index.analysis.AnalysisRegistry; +import org.opensearch.script.ScriptService; import org.opensearch.search.pipeline.Processor; import org.opensearch.search.pipeline.SearchPhaseResultsProcessor; +import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.pipeline.SearchRequestProcessor; import org.opensearch.search.pipeline.SearchResponseProcessor; +import org.opensearch.threadpool.Scheduler; import java.util.Collections; import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.LongSupplier; /** * An extension point for {@link Plugin} implementation to add custom search pipeline processors. @@ -29,7 +40,7 @@ public interface SearchPipelinePlugin { * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory} * to create the processor from a given pipeline configuration. */ - default Map> getRequestProcessors(Processor.Parameters parameters) { + default Map> getRequestProcessors(Parameters parameters) { return Collections.emptyMap(); } @@ -40,7 +51,7 @@ default Map> getRequestProcess * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory} * to create the processor from a given pipeline configuration. */ - default Map> getResponseProcessors(Processor.Parameters parameters) { + default Map> getResponseProcessors(Parameters parameters) { return Collections.emptyMap(); } @@ -51,7 +62,78 @@ default Map> getResponseProce * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory} * to create the processor from a given pipeline configuration. */ - default Map> getSearchPhaseResultsProcessors(Processor.Parameters parameters) { + default Map> getSearchPhaseResultsProcessors(Parameters parameters) { return Collections.emptyMap(); } + + /** + * Infrastructure class that holds services that can be used by processor factories to create processor instances + * and that gets passed around to all {@link SearchPipelinePlugin}s. + */ + class Parameters { + + /** + * Useful to provide access to the node's environment like config directory to processor factories. + */ + public final Environment env; + + /** + * Provides processors script support. + */ + public final ScriptService scriptService; + + /** + * Provide analyzer support + */ + public final AnalysisRegistry analysisRegistry; + + /** + * Allows processors to read headers set by {@link org.opensearch.action.support.ActionFilter} + * instances that have run while handling the current search. + */ + public final ThreadContext threadContext; + + public final LongSupplier relativeTimeSupplier; + + public final SearchPipelineService searchPipelineService; + + public final Consumer genericExecutor; + + public final NamedXContentRegistry namedXContentRegistry; + + /** + * Provides scheduler support + */ + public final BiFunction scheduler; + + /** + * Provides access to the node's cluster client + */ + public final Client client; + + public Parameters( + Environment env, + ScriptService scriptService, + AnalysisRegistry analysisRegistry, + ThreadContext threadContext, + LongSupplier relativeTimeSupplier, + BiFunction scheduler, + SearchPipelineService searchPipelineService, + Client client, + Consumer genericExecutor, + NamedXContentRegistry namedXContentRegistry + ) { + this.env = env; + this.scriptService = scriptService; + this.threadContext = threadContext; + this.analysisRegistry = analysisRegistry; + this.relativeTimeSupplier = relativeTimeSupplier; + this.scheduler = scheduler; + this.searchPipelineService = searchPipelineService; + this.client = client; + this.genericExecutor = genericExecutor; + this.namedXContentRegistry = namedXContentRegistry; + } + + } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java index 612e979e56070..060894a37e5ed 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java @@ -77,19 +77,28 @@ static PipelineWithMetrics create( Map> phaseResultsProcessorFactories, NamedWriteableRegistry namedWriteableRegistry, OperationMetrics totalRequestProcessingMetrics, - OperationMetrics totalResponseProcessingMetrics + OperationMetrics totalResponseProcessingMetrics, + Processor.PipelineContext pipelineContext ) throws Exception { String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY); Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null); List> requestProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, REQUEST_PROCESSORS_KEY); - List requestProcessors = readProcessors(requestProcessorFactories, requestProcessorConfigs); + List requestProcessors = readProcessors( + requestProcessorFactories, + requestProcessorConfigs, + pipelineContext + ); List> responseProcessorConfigs = ConfigurationUtils.readOptionalList( null, null, config, RESPONSE_PROCESSORS_KEY ); - List responseProcessors = readProcessors(responseProcessorFactories, responseProcessorConfigs); + List responseProcessors = readProcessors( + responseProcessorFactories, + responseProcessorConfigs, + pipelineContext + ); List> phaseResultsProcessorConfigs = ConfigurationUtils.readOptionalList( null, null, @@ -98,7 +107,8 @@ static PipelineWithMetrics create( ); List phaseResultsProcessors = readProcessors( phaseResultsProcessorFactories, - phaseResultsProcessorConfigs + phaseResultsProcessorConfigs, + pipelineContext ); if (config.isEmpty() == false) { throw new OpenSearchParseException( @@ -125,7 +135,8 @@ static PipelineWithMetrics create( private static List readProcessors( Map> processorFactories, - List> requestProcessorConfigs + List> requestProcessorConfigs, + Processor.PipelineContext pipelineContext ) throws Exception { List processors = new ArrayList<>(); if (requestProcessorConfigs == null) { @@ -140,7 +151,19 @@ private static List readProcessors( Map config = (Map) entry.getValue(); String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); - processors.add(processorFactories.get(type).create(processorFactories, tag, description, config)); + processors.add(processorFactories.get(type).create(processorFactories, tag, description, config, pipelineContext)); + if (config.isEmpty() == false) { + String processorName = type; + if (tag != null) { + processorName = processorName + ":" + tag; + } + throw new OpenSearchParseException( + "processor [" + + processorName + + "] doesn't support one or more provided configuration parameters: " + + Arrays.toString(config.keySet().toArray()) + ); + } } } return Collections.unmodifiableList(processors); diff --git a/server/src/main/java/org/opensearch/search/pipeline/Processor.java b/server/src/main/java/org/opensearch/search/pipeline/Processor.java index ee28db1cc334d..cc96132479c74 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/Processor.java +++ b/server/src/main/java/org/opensearch/search/pipeline/Processor.java @@ -8,19 +8,7 @@ package org.opensearch.search.pipeline; -import org.opensearch.client.Client; -import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.env.Environment; -import org.opensearch.index.analysis.AnalysisRegistry; -import org.opensearch.plugins.SearchPipelinePlugin; -import org.opensearch.script.ScriptService; -import org.opensearch.threadpool.Scheduler; - import java.util.Map; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import java.util.function.LongSupplier; /** * A processor implementation may modify the request or response from a search call. @@ -33,6 +21,12 @@ * @opensearch.internal */ public interface Processor { + /** + * Processor configuration key to let the factory know the context for pipeline creation. + *

+ * See {@link PipelineSource}. + */ + String PIPELINE_SOURCE = "pipeline_source"; /** * Gets the type of processor @@ -61,81 +55,45 @@ interface Factory { * @param tag The tag for the processor * @param description A short description of what this processor does * @param config The configuration for the processor - * * Note: Implementations are responsible for removing the used configuration * keys, so that after creation the config map should be empty. + * @param pipelineContext Contextual information about the enclosing pipeline. */ - T create(Map> processorFactories, String tag, String description, Map config) throws Exception; + T create( + Map> processorFactories, + String tag, + String description, + Map config, + PipelineContext pipelineContext + ) throws Exception; } /** - * Infrastructure class that holds services that can be used by processor factories to create processor instances - * and that gets passed around to all {@link SearchPipelinePlugin}s. + * Contextual information about the enclosing pipeline. A processor factory may change processor initialization behavior or + * pass this information to the created processor instance. */ - class Parameters { + class PipelineContext { + private final PipelineSource pipelineSource; - /** - * Useful to provide access to the node's environment like config directory to processor factories. - */ - public final Environment env; - - /** - * Provides processors script support. - */ - public final ScriptService scriptService; - - /** - * Provide analyzer support - */ - public final AnalysisRegistry analysisRegistry; - - /** - * Allows processors to read headers set by {@link org.opensearch.action.support.ActionFilter} - * instances that have run while handling the current search. - */ - public final ThreadContext threadContext; - - public final LongSupplier relativeTimeSupplier; - - public final SearchPipelineService searchPipelineService; - - public final Consumer genericExecutor; - - public final NamedXContentRegistry namedXContentRegistry; - - /** - * Provides scheduler support - */ - public final BiFunction scheduler; - - /** - * Provides access to the node's cluster client - */ - public final Client client; + public PipelineContext(PipelineSource pipelineSource) { + this.pipelineSource = pipelineSource; + } - public Parameters( - Environment env, - ScriptService scriptService, - AnalysisRegistry analysisRegistry, - ThreadContext threadContext, - LongSupplier relativeTimeSupplier, - BiFunction scheduler, - SearchPipelineService searchPipelineService, - Client client, - Consumer genericExecutor, - NamedXContentRegistry namedXContentRegistry - ) { - this.env = env; - this.scriptService = scriptService; - this.threadContext = threadContext; - this.analysisRegistry = analysisRegistry; - this.relativeTimeSupplier = relativeTimeSupplier; - this.scheduler = scheduler; - this.searchPipelineService = searchPipelineService; - this.client = client; - this.genericExecutor = genericExecutor; - this.namedXContentRegistry = namedXContentRegistry; + public PipelineSource getPipelineSource() { + return pipelineSource; } + } + /** + * A processor factory may change the processor initialization behavior based on the creation context (e.g. avoiding + * creating expensive resources during validation or in a request-scoped pipeline.) + */ + enum PipelineSource { + // A named pipeline is being created or updated + UPDATE_PIPELINE, + // Pipeline is defined within a search request + SEARCH_REQUEST, + // A named pipeline is being validated before being written to cluster state + VALIDATE_PIPELINE } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 70dc8546a077f..83a7a0564467e 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -103,7 +103,7 @@ public SearchPipelineService( this.scriptService = scriptService; this.threadPool = threadPool; this.namedWriteableRegistry = namedWriteableRegistry; - Processor.Parameters parameters = new Processor.Parameters( + SearchPipelinePlugin.Parameters parameters = new SearchPipelinePlugin.Parameters( env, scriptService, analysisRegistry, @@ -189,7 +189,8 @@ void innerUpdatePipelines(SearchPipelineMetadata newSearchPipelineMetadata) { phaseInjectorProcessorFactories, namedWriteableRegistry, totalRequestProcessingMetrics, - totalResponseProcessingMetrics + totalResponseProcessingMetrics, + new Processor.PipelineContext(Processor.PipelineSource.UPDATE_PIPELINE) ); newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, newPipeline)); @@ -289,7 +290,8 @@ void validatePipeline(Map searchPipelineInfos phaseInjectorProcessorFactories, namedWriteableRegistry, new OperationMetrics(), // Use ephemeral metrics for validation - new OperationMetrics() + new OperationMetrics(), + new Processor.PipelineContext(Processor.PipelineSource.VALIDATE_PIPELINE) ); List exceptions = new ArrayList<>(); for (SearchRequestProcessor processor : pipeline.getSearchRequestProcessors()) { @@ -388,7 +390,8 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) { phaseInjectorProcessorFactories, namedWriteableRegistry, totalRequestProcessingMetrics, - totalResponseProcessingMetrics + totalResponseProcessingMetrics, + new Processor.PipelineContext(Processor.PipelineSource.SEARCH_REQUEST) ); } catch (Exception e) { throw new SearchPipelineProcessingException(e); diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index 2ac0b2136ddd9..84f39e4bdab42 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -77,19 +77,17 @@ public class SearchPipelineServiceTests extends OpenSearchTestCase { private static final SearchPipelinePlugin DUMMY_PLUGIN = new SearchPipelinePlugin() { @Override - public Map> getRequestProcessors(Processor.Parameters parameters) { - return Map.of("foo", (factories, tag, description, config) -> null); + public Map> getRequestProcessors(Parameters parameters) { + return Map.of("foo", (factories, tag, description, config, ctx) -> null); } - public Map> getResponseProcessors(Processor.Parameters parameters) { - return Map.of("bar", (factories, tag, description, config) -> null); + public Map> getResponseProcessors(Parameters parameters) { + return Map.of("bar", (factories, tag, description, config, ctx) -> null); } @Override - public Map> getSearchPhaseResultsProcessors( - Processor.Parameters parameters - ) { - return Map.of("zoe", (factories, tag, description, config) -> null); + public Map> getSearchPhaseResultsProcessors(Parameters parameters) { + return Map.of("zoe", (factories, tag, description, config, ctx) -> null); } }; @@ -303,7 +301,7 @@ public SearchPhaseName getAfterPhase() { private SearchPipelineService createWithProcessors() { Map> requestProcessors = new HashMap<>(); - requestProcessors.put("scale_request_size", (processorFactories, tag, description, config) -> { + requestProcessors.put("scale_request_size", (processorFactories, tag, description, config, ctx) -> { float scale = ((Number) config.remove("scale")).floatValue(); return new FakeRequestProcessor( "scale_request_size", @@ -313,13 +311,13 @@ private SearchPipelineService createWithProcessors() { ); }); Map> responseProcessors = new HashMap<>(); - responseProcessors.put("fixed_score", (processorFactories, tag, description, config) -> { + responseProcessors.put("fixed_score", (processorFactories, tag, description, config, ctx) -> { float score = ((Number) config.remove("score")).floatValue(); return new FakeResponseProcessor("fixed_score", tag, description, rsp -> rsp.getHits().forEach(h -> h.score(score))); }); Map> searchPhaseProcessors = new HashMap<>(); - searchPhaseProcessors.put("max_score", (processorFactories, tag, description, config) -> { + searchPhaseProcessors.put("max_score", (processorFactories, tag, description, config, context) -> { final float finalScore = config.containsKey("score") ? ((Number) config.remove("score")).floatValue() : 100f; final Consumer querySearchResultConsumer = (result) -> result.queryResult().topDocs().maxScore = finalScore; return new FakeSearchPhaseResultsProcessor("max_score", tag, description, querySearchResultConsumer); @@ -354,19 +352,17 @@ private SearchPipelineService createWithProcessors( this.writableRegistry(), Collections.singletonList(new SearchPipelinePlugin() { @Override - public Map> getRequestProcessors(Processor.Parameters parameters) { + public Map> getRequestProcessors(Parameters parameters) { return requestProcessors; } @Override - public Map> getResponseProcessors(Processor.Parameters parameters) { + public Map> getResponseProcessors(Parameters parameters) { return responseProcessors; } @Override - public Map> getSearchPhaseResultsProcessors( - Processor.Parameters parameters - ) { + public Map> getSearchPhaseResultsProcessors(Parameters parameters) { return phaseProcessors; } @@ -897,10 +893,9 @@ public void testInfo() { } public void testExceptionOnPipelineCreation() { - Map> badFactory = Map.of( - "bad_factory", - (pf, t, f, c) -> { throw new RuntimeException(); } - ); + Map> badFactory = Map.of("bad_factory", (pf, t, f, c, ctx) -> { + throw new RuntimeException(); + }); SearchPipelineService searchPipelineService = createWithProcessors(badFactory, Collections.emptyMap(), Collections.emptyMap()); Map pipelineSourceMap = new HashMap<>(); @@ -920,7 +915,7 @@ public void testExceptionOnRequestProcessing() { }); Map> throwingRequestProcessorFactory = Map.of( "throwing_request", - (pf, t, f, c) -> throwingRequestProcessor + (pf, t, f, c, ctx) -> throwingRequestProcessor ); SearchPipelineService searchPipelineService = createWithProcessors( @@ -945,7 +940,7 @@ public void testExceptionOnResponseProcessing() throws Exception { }); Map> throwingResponseProcessorFactory = Map.of( "throwing_response", - (pf, t, f, c) -> throwingResponseProcessor + (pf, t, f, c, ctx) -> throwingResponseProcessor ); SearchPipelineService searchPipelineService = createWithProcessors( @@ -955,7 +950,7 @@ public void testExceptionOnResponseProcessing() throws Exception { ); Map pipelineSourceMap = new HashMap<>(); - pipelineSourceMap.put(Pipeline.RESPONSE_PROCESSORS_KEY, List.of(Map.of("throwing_response", Collections.emptyMap()))); + pipelineSourceMap.put(Pipeline.RESPONSE_PROCESSORS_KEY, List.of(Map.of("throwing_response", new HashMap<>()))); SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).searchPipelineSource(pipelineSourceMap); SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); @@ -973,18 +968,18 @@ public void testStats() throws Exception { }); Map> requestProcessors = Map.of( "successful_request", - (pf, t, f, c) -> new FakeRequestProcessor("successful_request", "2", null, r -> {}), + (pf, t, f, c, ctx) -> new FakeRequestProcessor("successful_request", "2", null, r -> {}), "throwing_request", - (pf, t, f, c) -> throwingRequestProcessor + (pf, t, f, c, ctx) -> throwingRequestProcessor ); SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", "3", null, r -> { throw new RuntimeException(); }); Map> responseProcessors = Map.of( "successful_response", - (pf, t, f, c) -> new FakeResponseProcessor("successful_response", "4", null, r -> {}), + (pf, t, f, c, ctx) -> new FakeResponseProcessor("successful_response", "4", null, r -> {}), "throwing_response", - (pf, t, f, c) -> throwingResponseProcessor + (pf, t, f, c, ctx) -> throwingResponseProcessor ); SearchPipelineService searchPipelineService = createWithProcessors(requestProcessors, responseProcessors, Collections.emptyMap()); @@ -1088,4 +1083,64 @@ private static void assertPipelineStats(OperationStats stats, long count, long f assertEquals(stats.getCount(), count); assertEquals(stats.getFailedCount(), failed); } + + public void testAdHocRejectingProcessor() { + String processorType = "ad_hoc_rejecting"; + Map> requestProcessorFactories = Map.of(processorType, (pf, t, d, c, ctx) -> { + if (ctx.getPipelineSource() == Processor.PipelineSource.SEARCH_REQUEST) { + throw new IllegalArgumentException(processorType + " cannot be created as part of a pipeline defined in a search request"); + } + return new FakeRequestProcessor(processorType, t, d, r -> {}); + }); + + SearchPipelineService searchPipelineService = createWithProcessors( + requestProcessorFactories, + Collections.emptyMap(), + Collections.emptyMap() + ); + + String id = "_id"; + SearchPipelineService.PipelineHolder pipeline = searchPipelineService.getPipelines().get(id); + assertNull(pipeline); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + PutSearchPipelineRequest putRequest = new PutSearchPipelineRequest( + id, + new BytesArray("{\"request_processors\":[" + " { \"" + processorType + "\": {}}" + "]}"), + XContentType.JSON + ); + ClusterState previousClusterState = clusterState; + clusterState = SearchPipelineService.innerPut(putRequest, clusterState); + // The following line successfully creates the pipeline: + searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); + + Map pipelineSourceMap = new HashMap<>(); + pipelineSourceMap.put(Pipeline.REQUEST_PROCESSORS_KEY, List.of(Map.of(processorType, Collections.emptyMap()))); + + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().searchPipelineSource(pipelineSourceMap); + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + expectThrows(SearchPipelineProcessingException.class, () -> searchPipelineService.resolvePipeline(searchRequest)); + } + + public void testExtraParameterInProcessorConfig() { + SearchPipelineService searchPipelineService = createWithProcessors(); + + Map pipelineSourceMap = new HashMap<>(); + Map processorConfig = new HashMap<>( + Map.of("score", 1.0f, "tag", "my_tag", "comment", "I just like to add extra parameters so that I feel like I'm being heard.") + ); + pipelineSourceMap.put(Pipeline.RESPONSE_PROCESSORS_KEY, List.of(Map.of("fixed_score", processorConfig))); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().searchPipelineSource(pipelineSourceMap); + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + try { + searchPipelineService.resolvePipeline(searchRequest); + fail("Exception should have been thrown"); + } catch (SearchPipelineProcessingException e) { + assertTrue( + e.getMessage() + .contains("processor [fixed_score:my_tag] doesn't support one or more provided configuration parameters: [comment]") + ); + } catch (Exception e) { + fail("Wrong exception type: " + e.getClass()); + } + } } From a0da953e33091bcd45b773f2e41cbfcdf4b5a813 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 6 Jul 2023 17:14:59 -0400 Subject: [PATCH 061/186] Update to Gradle 8.2 (#8018) Signed-off-by: Andriy Redko --- benchmarks/build.gradle | 10 ++++-- build.gradle | 6 ++-- buildSrc/build.gradle | 33 ++++++++++-------- buildSrc/reaper/build.gradle | 6 ++-- .../gradle/plugin/PluginBuildPlugin.groovy | 12 +++++-- .../org/opensearch/gradle/PublishPlugin.java | 4 +-- .../precommit/ForbiddenPatternsTask.java | 6 ++-- .../gradle/precommit/LoggerUsageTask.java | 6 ++-- .../gradle/test/rest/CopyRestApiTask.java | 4 +-- .../gradle/test/rest/CopyRestTestsTask.java | 4 +-- .../opensearch/gradle/util/GradleUtils.java | 4 +-- .../java/org/opensearch/gradle/util/Util.java | 6 ++-- client/benchmark/build.gradle | 10 ++++-- client/rest-high-level/build.gradle | 6 ++-- client/rest/build.gradle | 12 ++++--- client/sniffer/build.gradle | 12 ++++--- client/test/build.gradle | 10 ++++-- .../archives/integ-test-zip/build.gradle | 6 ++-- .../tools/java-version-checker/build.gradle | 6 ++-- distribution/tools/launchers/build.gradle | 4 ++- distribution/tools/plugin-cli/build.gradle | 4 ++- distribution/tools/upgrade-cli/build.gradle | 4 ++- doc-tools/build.gradle | 6 ++-- gradle.properties | 3 ++ gradle/code-coverage.gradle | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 62076 -> 63375 bytes gradle/wrapper/gradle-wrapper.properties | 4 +-- gradlew | 5 ++- libs/common/build.gradle | 4 ++- libs/core/build.gradle | 4 ++- modules/lang-painless/build.gradle | 2 +- modules/lang-painless/spi/build.gradle | 6 ++-- server/build.gradle | 4 ++- settings.gradle | 2 +- 34 files changed, 139 insertions(+), 78 deletions(-) diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 4a7825e9ba35b..02aa9319cc583 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -32,10 +32,16 @@ import org.opensearch.gradle.info.BuildParams apply plugin: 'opensearch.build' apply plugin: 'application' -mainClassName = 'org.openjdk.jmh.Main' assemble.enabled = false -archivesBaseName = 'opensearch-benchmarks' + +application { + mainClass = 'org.openjdk.jmh.Main' +} + +base { + archivesBaseName = 'opensearch-benchmarks' +} test.enabled = false diff --git a/build.gradle b/build.gradle index eec01e840149e..ca4c6c3635d57 100644 --- a/build.gradle +++ b/build.gradle @@ -327,7 +327,7 @@ allprojects { javadoc.options.addStringOption('Xwerror', '-quiet') } javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"] - javadoc.options.addStringOption("-release", targetCompatibility.majorVersion) + javadoc.options.addStringOption("-release", java.targetCompatibility.majorVersion) } // support for reproducible builds @@ -375,7 +375,7 @@ allprojects { } else { // Link to non-shadowed dependant projects project.javadoc.dependsOn "${upstreamProject.path}:javadoc" - String externalLinkName = upstreamProject.archivesBaseName + String externalLinkName = upstreamProject.base.archivesBaseName String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version String projectRelativePath = project.relativePath(upstreamProject.buildDir) project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/" @@ -452,7 +452,7 @@ gradle.projectsEvaluated { testReportAggregation it } subprojects.findAll { it.pluginManager.hasPlugin('jacoco') }.forEach { - jacocoAggregation it + jacocoAggregation it } } } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 47d4747d4b9a6..feb8da7c20984 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -34,7 +34,6 @@ import org.gradle.util.GradleVersion plugins { id 'java-gradle-plugin' id 'groovy' - id 'java-test-fixtures' } group = 'org.opensearch.gradle' @@ -79,9 +78,17 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) { } sourceSets { + test { + java { + srcDirs += ['src/testFixtures/java'] + } + } integTest { compileClasspath += sourceSets["main"].output + configurations["testRuntimeClasspath"] runtimeClasspath += output + compileClasspath + java { + srcDirs += ['src/testFixtures/java'] + } } } @@ -111,7 +118,7 @@ dependencies { api 'org.apache.rat:apache-rat:0.15' api 'commons-io:commons-io:2.13.0' api "net.java.dev.jna:jna:5.13.0" - api 'gradle.plugin.com.github.johnrengelman:shadow:8.0.0' + api 'com.github.johnrengelman:shadow:8.1.1' api 'org.jdom:jdom2:2.0.6.1' api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}" api 'de.thetaphi:forbiddenapis:3.5.1' @@ -124,10 +131,10 @@ dependencies { api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" api "org.ajoberstar.grgit:grgit-core:5.2.0" - testFixturesApi "junit:junit:${props.getProperty('junit')}" - testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" - testFixturesApi gradleApi() - testFixturesApi gradleTestKit() + testImplementation "junit:junit:${props.getProperty('junit')}" + testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" + testRuntimeOnly gradleApi() + testRuntimeOnly gradleTestKit() testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.35.0' testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}" integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') { @@ -164,8 +171,10 @@ if (project != rootProject) { apply plugin: 'opensearch.publish' allprojects { - targetCompatibility = JavaVersion.VERSION_11 - sourceCompatibility = JavaVersion.VERSION_11 + java { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 + } } // groovydoc succeeds, but has some weird internal exception... @@ -174,7 +183,7 @@ if (project != rootProject) { // build-tools is not ready for primetime with these... tasks.named("dependencyLicenses").configure { it.enabled = false } dependenciesInfo.enabled = false - disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures') + disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest') jarHell.enabled = false thirdPartyAudit.enabled = false if (org.opensearch.gradle.info.BuildParams.inFipsJvm) { @@ -241,12 +250,6 @@ if (project != rootProject) { } } - // disable fail-on-warnings for this specific task which trips Java 11 bug - // https://bugs.openjdk.java.net/browse/JDK-8209058 - tasks.named("compileTestFixturesJava").configure { - options.compilerArgs -= '-Werror' - } - tasks.register("integTest", Test) { inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE) systemProperty 'test.version_under_test', version diff --git a/buildSrc/reaper/build.gradle b/buildSrc/reaper/build.gradle index 4ccbec894e30e..58d06b02e9f4b 100644 --- a/buildSrc/reaper/build.gradle +++ b/buildSrc/reaper/build.gradle @@ -11,8 +11,10 @@ apply plugin: 'java' -targetCompatibility = JavaVersion.VERSION_11 -sourceCompatibility = JavaVersion.VERSION_11 +java { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} jar { archiveFileName = "${project.name}.jar" diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy index 2428d9e2fa4fa..556763333d279 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy @@ -44,6 +44,8 @@ import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.plugins.BasePlugin +import org.gradle.api.plugins.BasePluginExtension +import org.gradle.api.plugins.JavaPluginExtension import org.gradle.api.publish.maven.MavenPublication import org.gradle.api.publish.maven.plugins.MavenPublishPlugin import org.gradle.api.publish.maven.tasks.GenerateMavenPom @@ -85,7 +87,9 @@ class PluginBuildPlugin implements Plugin { PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) configurePublishing(project, extension1) String name = extension1.name - project.archivesBaseName = name + + BasePluginExtension base = project.getExtensions().findByType(BasePluginExtension.class) + base.archivesBaseName = name project.description = extension1.description if (extension1.name == null) { @@ -98,12 +102,13 @@ class PluginBuildPlugin implements Plugin { throw new InvalidUserDataException('classname is a required setting for opensearchplugin') } + JavaPluginExtension java = project.getExtensions().findByType(JavaPluginExtension.class) Map properties = [ 'name' : extension1.name, 'description' : extension1.description, 'version' : extension1.version, 'opensearchVersion' : Version.fromString(VersionProperties.getOpenSearch()).toString(), - 'javaVersion' : project.targetCompatibility as String, + 'javaVersion' : java.targetCompatibility as String, 'classname' : extension1.classname, 'customFolderName' : extension1.customFolderName, 'extendedPlugins' : extension1.extendedPlugins.join(','), @@ -156,8 +161,9 @@ class PluginBuildPlugin implements Plugin { } // always configure publishing for client jars project.publishing.publications.nebula(MavenPublication).artifactId(extension.name + "-client") + final BasePluginExtension base = project.getExtensions().findByType(BasePluginExtension.class) project.tasks.withType(GenerateMavenPom.class).configureEach { GenerateMavenPom generatePOMTask -> - generatePOMTask.destination = "${project.buildDir}/distributions/${project.archivesBaseName}-client-${project.versions.opensearch}.pom" + generatePOMTask.destination = "${project.buildDir}/distributions/${base.archivesBaseName}-client-${project.versions.opensearch}.pom" } } else { if (project.plugins.hasPlugin(MavenPublishPlugin)) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java index 1015a2dfb7d72..97e923c366598 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java @@ -46,7 +46,7 @@ import org.gradle.api.XmlProvider; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.plugins.BasePlugin; -import org.gradle.api.plugins.BasePluginConvention; +import org.gradle.api.plugins.BasePluginExtension; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.publish.PublishingExtension; import org.gradle.api.publish.maven.MavenPublication; @@ -77,7 +77,7 @@ public void apply(Project project) { } private static String getArchivesBaseName(Project project) { - return project.getConvention().getPlugin(BasePluginConvention.class).getArchivesBaseName(); + return project.getExtensions().getByType(BasePluginExtension.class).getArchivesBaseName(); } /**Configuration generation of maven poms. */ diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java index 754743b9b784c..6ef1e77f5138f 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java @@ -36,7 +36,7 @@ import org.gradle.api.InvalidUserDataException; import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileTree; -import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.IgnoreEmptyDirectories; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; @@ -106,8 +106,8 @@ public ForbiddenPatternsTask() { @IgnoreEmptyDirectories @PathSensitive(PathSensitivity.RELATIVE) public FileCollection getFiles() { - return getProject().getConvention() - .getPlugin(JavaPluginConvention.class) + return getProject().getExtensions() + .getByType(JavaPluginExtension.class) .getSourceSets() .stream() .map(sourceSet -> sourceSet.getAllSource().matching(filesFilter)) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java index ff9f6619d64e6..db215fb65ef95 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java @@ -34,7 +34,7 @@ import org.opensearch.gradle.LoggedExec; import org.gradle.api.file.FileCollection; -import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.IgnoreEmptyDirectories; @@ -82,8 +82,8 @@ public void setClasspath(FileCollection classpath) { @SkipWhenEmpty @IgnoreEmptyDirectories public FileCollection getClassDirectories() { - return getProject().getConvention() - .getPlugin(JavaPluginConvention.class) + return getProject().getExtensions() + .getByType(JavaPluginExtension.class) .getSourceSets() .stream() // Don't pick up all source sets like the java9 ones as logger-check doesn't support the class format diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java index 1468c4cb1b537..0275664276877 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java @@ -41,7 +41,7 @@ import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; -import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.ListProperty; import org.gradle.api.tasks.IgnoreEmptyDirectories; import org.gradle.api.tasks.Input; @@ -240,7 +240,7 @@ private File getTestOutputResourceDir() { private Optional getSourceSet() { Project project = getProject(); - return project.getConvention().findPlugin(JavaPluginConvention.class) == null + return project.getExtensions().findByType(JavaPluginExtension.class) == null ? Optional.empty() : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(getSourceSetName())); } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java index dd94d040cb9d8..ebd6c49fd6157 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java @@ -41,7 +41,7 @@ import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; -import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.ListProperty; import org.gradle.api.tasks.IgnoreEmptyDirectories; import org.gradle.api.tasks.Input; @@ -178,7 +178,7 @@ void copy() { private Optional getSourceSet() { Project project = getProject(); - return project.getConvention().findPlugin(JavaPluginConvention.class) == null + return project.getExtensions().findByType(JavaPluginExtension.class) == null ? Optional.empty() : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(getSourceSetName())); } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java b/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java index 054f01788d126..031fee2d1127f 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java @@ -40,7 +40,7 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.plugins.JavaBasePlugin; -import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.Provider; import org.gradle.api.services.BuildService; import org.gradle.api.services.BuildServiceRegistration; @@ -68,7 +68,7 @@ public static Action noop() { } public static SourceSetContainer getJavaSourceSets(Project project) { - return project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets(); + return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets(); } public static TaskProvider maybeRegister(TaskContainer tasks, String name, Class clazz, Action action) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/util/Util.java b/buildSrc/src/main/java/org/opensearch/gradle/util/Util.java index 71b1e5040340d..a4d2c59cf8cad 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/util/Util.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/util/Util.java @@ -37,7 +37,7 @@ import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.file.FileTree; -import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.util.PatternFilterable; @@ -149,7 +149,7 @@ public static FileTree getJavaTestAndMainSourceResources(Project project, Action * @return An Optional that contains the Java test SourceSet if it exists. */ public static Optional getJavaTestSourceSet(Project project) { - return project.getConvention().findPlugin(JavaPluginConvention.class) == null + return project.getExtensions().findByType(JavaPluginExtension.class) == null ? Optional.empty() : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.TEST_SOURCE_SET_NAME)); } @@ -159,7 +159,7 @@ public static Optional getJavaTestSourceSet(Project project) { * @return An Optional that contains the Java main SourceSet if it exists. */ public static Optional getJavaMainSourceSet(Project project) { - return project.getConvention().findPlugin(JavaPluginConvention.class) == null + return project.getExtensions().findByType(JavaPluginExtension.class) == null ? Optional.empty() : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME)); } diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle index 4aa4d7171e366..6fd5262f0ab4f 100644 --- a/client/benchmark/build.gradle +++ b/client/benchmark/build.gradle @@ -31,13 +31,17 @@ apply plugin: 'opensearch.build' apply plugin: 'application' -group = 'org.opensearch.client' +base { + group = 'org.opensearch.client' + archivesBaseName = 'client-benchmarks' +} // Not published so no need to assemble assemble.enabled = true -archivesBaseName = 'client-benchmarks' -mainClassName = 'org.opensearch.client.benchmark.BenchmarkMain' +application { + mainClass = 'org.opensearch.client.benchmark.BenchmarkMain' +} // never try to invoke tests on the benchmark project - there aren't any test.enabled = false diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 43758560b2a15..7daf4761b9310 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -37,8 +37,10 @@ apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.publish' apply plugin: 'opensearch.rest-resources' -group = 'org.opensearch.client' -archivesBaseName = 'opensearch-rest-high-level-client' +base { + group = 'org.opensearch.client' + archivesBaseName = 'opensearch-rest-high-level-client' +} restResources { //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 9ea7ad4ddb964..2c437c909fb03 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -33,11 +33,15 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' -targetCompatibility = JavaVersion.VERSION_11 -sourceCompatibility = JavaVersion.VERSION_11 +java { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} -group = 'org.opensearch.client' -archivesBaseName = 'opensearch-rest-client' +base { + group = 'org.opensearch.client' + archivesBaseName = 'opensearch-rest-client' +} dependencies { api "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 9823bc9afd347..f645b2dbbc933 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -30,11 +30,15 @@ apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' -targetCompatibility = JavaVersion.VERSION_11 -sourceCompatibility = JavaVersion.VERSION_11 +java { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} -group = 'org.opensearch.client' -archivesBaseName = 'opensearch-rest-client-sniffer' +base { + group = 'org.opensearch.client' + archivesBaseName = 'opensearch-rest-client-sniffer' +} dependencies { api project(":client:rest") diff --git a/client/test/build.gradle b/client/test/build.gradle index 13e9bd6b9e34a..f81a009389681 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -29,10 +29,14 @@ */ apply plugin: 'opensearch.build' -targetCompatibility = JavaVersion.VERSION_11 -sourceCompatibility = JavaVersion.VERSION_11 +java { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} -group = "${group}.client.test" +base { + group = "${group}.client.test" +} dependencies { api "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle index f3f848797961c..9418223b0a44d 100644 --- a/distribution/archives/integ-test-zip/build.gradle +++ b/distribution/archives/integ-test-zip/build.gradle @@ -36,7 +36,10 @@ apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.publish' apply plugin: 'com.netflix.nebula.maven-publish' -group = "org.opensearch.distribution.integ-test-zip" +base { + group = "org.opensearch.distribution.integ-test-zip" + archivesBaseName = "opensearch" +} integTest { dependsOn assemble @@ -48,7 +51,6 @@ processTestResources { } // make the pom file name use opensearch instead of the project name -archivesBaseName = "opensearch" ext.buildDist = parent.tasks.named("buildIntegTestZip") publishing { diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/java-version-checker/build.gradle index 9480a86ce6fb7..7fd16b910b293 100644 --- a/distribution/tools/java-version-checker/build.gradle +++ b/distribution/tools/java-version-checker/build.gradle @@ -11,8 +11,10 @@ apply plugin: 'opensearch.build' -sourceCompatibility = JavaVersion.VERSION_11 -targetCompatibility = JavaVersion.VERSION_11 +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} // targetting very old java versions enables a warning by default on newer JDK: disable it. compileJava.options.compilerArgs += '-Xlint:-options' diff --git a/distribution/tools/launchers/build.gradle b/distribution/tools/launchers/build.gradle index 7ebe5c7e64416..e75267f7c4a74 100644 --- a/distribution/tools/launchers/build.gradle +++ b/distribution/tools/launchers/build.gradle @@ -38,7 +38,9 @@ dependencies { testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" } -archivesBaseName = 'opensearch-launchers' +base { + archivesBaseName = 'opensearch-launchers' +} tasks.withType(CheckForbiddenApis).configureEach { replaceSignatureFiles 'jdk-signatures' diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 477f66625e124..e68fbf8d6fc3c 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -30,7 +30,9 @@ apply plugin: 'opensearch.build' -archivesBaseName = 'opensearch-plugin-cli' +base { + archivesBaseName = 'opensearch-plugin-cli' +} dependencies { compileOnly project(":server") diff --git a/distribution/tools/upgrade-cli/build.gradle b/distribution/tools/upgrade-cli/build.gradle index cb87faf7788fa..54d51936159ec 100644 --- a/distribution/tools/upgrade-cli/build.gradle +++ b/distribution/tools/upgrade-cli/build.gradle @@ -9,7 +9,9 @@ apply plugin: 'opensearch.build' -archivesBaseName = 'opensearch-upgrade-cli' +base { + archivesBaseName = 'opensearch-upgrade-cli' +} dependencies { compileOnly project(":server") diff --git a/doc-tools/build.gradle b/doc-tools/build.gradle index c47097c3d6035..e6ace21420dda 100644 --- a/doc-tools/build.gradle +++ b/doc-tools/build.gradle @@ -2,8 +2,10 @@ plugins { id 'java' } -group 'org.opensearch' -version '1.0.0-SNAPSHOT' +base { + group 'org.opensearch' + version '1.0.0-SNAPSHOT' +} repositories { mavenCentral() diff --git a/gradle.properties b/gradle.properties index 73df0940ce181..7c359ed2b652c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,6 +21,9 @@ org.gradle.jvmargs=-Xmx3g -XX:+HeapDumpOnOutOfMemoryError -Xss2m \ --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED options.forkOptions.memoryMaximumSize=3g +# Disable Gradle Enterprise Gradle plugin's test retry +systemProp.gradle.enterprise.testretry.enabled=false + # Disable duplicate project id detection # See https://docs.gradle.org/current/userguide/upgrading_version_6.html#duplicate_project_names_may_cause_publication_to_fail systemProp.org.gradle.dependency.duplicate.project.detection=false diff --git a/gradle/code-coverage.gradle b/gradle/code-coverage.gradle index d5a717b5ea720..dfb4ddba24113 100644 --- a/gradle/code-coverage.gradle +++ b/gradle/code-coverage.gradle @@ -19,7 +19,7 @@ repositories { allprojects { plugins.withId('jacoco') { - jacoco.toolVersion = '0.8.9' + jacoco.toolVersion = '0.8.10' } } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a79e29d3e0ab67b14947c167a862655af9b..033e24c4cdf41af1ab109bc7f253b2b887023340 100644 GIT binary patch delta 16170 zcmZv@1C%B~(=OPyZQHhOo71+Qo{!^7;G&o^S)+pkaqdJWHm~1r7od1qA}a4m7bN0H~O_TWh$Qcv`r+nb?b4TbS8d zxH6g9o4C29YUpd@YhrwdLs-IyGpjd3(n_D1EQ+2>M}EC_Qd^DMB&z+Y-R@$d*<|Y<~_L?8O}c#13DZ`CI-je^V*!p27iTh zVF^v_sc+#ATfG`o!(m-#)8OIgpcJaaK&dTtcz~bzH_spvFh(X~Nd=l%)i95)K-yk?O~JY-q9yJKyNwGpuUo601UzzZnZP2>f~C7ET%*JQ`7U^c%Ay= z*VXGhB(=zePs-uvej`1AV`+URCzI7opL{ct^|Lg3`JRQ#N2liRT0J3kn2{O5?+)Xh zg+2W4_vVGeL^tu5mNC*w+M@qOsA?i7Q5Y!W}0%`WElV9J|}=8*@{O1`1(!wCebWJz&EbIE09Ar_<&ldhsD}pR(~NfS=IJb>x%X z{2ulD!5`cb!w+v^IGu~jd3D$fUs>e3cW|v_Cm{8={NL)ZoxNQqikAB&nbiz7mbKz( zWjH73t*#;8Rv5%^+JhrK!zDSutNaUZF#xIcX-J?XTXJMUzc0+Q{3)Xt)KYbRR4)MYT4?1fDz4 z0NVFLz!!^q(*mC;cfO~%{B}A^V3|1aPPqpOYCO4o^)?p?Hn17_0AbdX$f;k!9sL^g z{n_Q5yM!yp{oU))sbp&r6v}Au6R`9Z#h@0oM&1n0>wAP27GtH zG#~tyCu38r+Xh)31z*ShTdXWfb`4h!sraW8_kR1VGraUOtA9}O2g{N$S+1{3q>z*< zDEs&xo6@|O7lJlzn%!gmnJL@mh6XY?H2^>+tYwAp2aD&ve*;dNlFRUUD4uJsz0s{jA0wM|`g_Bk- z2nGTI4FLio^iSgCYQ<~?w6VhgXuFy?J6pI)*tog7+L(H{+c-IDy4s67IsWSv-2ZoX zkgKk*j4q1tU51^udPJsziAoFE%s5Wgi({t%V=JasWm6hHcE*-AVByK0i}t9!4^NT& zYJ1?sHp;I5vxtJi@z=?8N5Bc2Rp96QJ7Pawo_W$pO{f?a?6fX`?dHe8J+yAg-F$LU zXmTjqP`_JciO)bHLs}L><&(2CORPpITFZ5y{Ha$rW};;c-n)RcD`TyHnL?)Fx{0?I zqQ|D4T`xLJy`A}h{D57UR@bD8{Bw{9rlPt&U?{4 zTbO4-nHnPS!as<)ecV@VpH~W*$zoPr8f09_MZBPjoU zamA5hmU=F0q4v*u)BvEyDNo)GJxs9tiPkp2uhlGLR2bUD{NSjGGCixR9?$LKAlsip zUIa{WQs#68GH3NL{(FUyk-k=lrtx{V24k>kq~uc+St1uH0Yf3s547xvD5T*@n^+VN zKO~$H#RFW+Sd*M?`&+A$L<%DwNmIW&h>4j}vyxu3PmHrGwp?hXJp!{^>$Ax2WY&9} z5fJvDKBT&~%2QWqTGf{=6Pv2U+0HUQRv9%RZLR`G^XNdKRZt`Zs z)vuUr#7C#oQ00KL7$M$(yHa*C4XZ~*t9NPMJU`fACD3v+wvLzMJipnOfRmh_kN5oD zZ;)G|-j$^OF~-yWW*p1m#1)%%tWgg_?ps;<cvxwa&b=_7Iu)xM#KIHR~gWVSQGmujR;bCgI%H#(_~8O`LAHbJ%9L?R(Dt zq%5@6HsP4(%%tF4t#7v$y&h*i|KihD+E^Q7n~`1KzELK>5I8-`H|JF2Cq9CgniYyS z_4op2_>b9Il(p8PquZ{h8Gy$%WA+8t)o_gCdb75|9NJ&}Y*D~a6)VE@eT3!qvvSPz z4-A4Vw^rS17uWVctor@Gky4eiT6nF=PVY~8jzjKM-GlQzF5I-V&Z7d^G3?o9`C9gHU5GOAMLIZIOBw|s--tIy=R#b8@3;?-9Y8jeFt`AhO z8tTwGxksHRNk>;%uqWW&Q!^M?CwVDvX-*wTji*J^X%}1`6Z(#9OsQQfUI9x&CAj=W z-tDF7TYPVS7zfx~aje8Z@J>er!E<@63gEY)W{b!AF%?j%VG;B3b;Kt6VVH0qxBLrC z*82l$taUKcm}zRM=K+>H%w7(10hX25ud7r}c#sEK;mnBsVbD;$qu_|UEarcuS7aYi zcMjgkjmj=#d&K?NX=qgouhsLh{iYTe8qtsU~kLwg4&&Q1YGyz6D@(-w< zl~tx6ulu}VfKZ@_gt2aL@E`A`ULme@K+ zek2hch6FNgHdbowNo)mBs0da-}bhPw|R1u{4 zEZ?T!7j&^lNPs1je%@Em^CPp$cX%GrCBn66>D{`Ugf%+~@)w+gX2xGJ1qCy6|1f8m zkW@0=CvkEuR0$mn*wuIvn?-qRMNjtj*c5Z_P}N^he{2=<@XK4^ zC{Zs89DIB6QjEE2PRx9Le^?_kvTpBWr~%L249F}8N&xTV?+_;?oyfV?V^T(ioIxw@ zYNZUlBAc=A{A709=R`$--jqG{jPQj-7f_Sr1$o&kapsFL3jBVIE*Z4&L}1ve?@wh=%eda^BRYm=>pJ z{p#Gotpa1aH^l+Oclp_+$Whjp_q3(G8zS<1;!#*67K0Du1}RQPo&G8mVeftaJ&a++ zYlh?j&;3LJA5Q4fDBsWauFn>VvG_9Tcrr2Yt-#+%rO0ST1GFitK8f10=rq|6lf1q? zZgVH$pWLo_(3QZ@KH}q%V;KT>r!K|?t?LSBWRUoPcv3to`%wC6ZRPF|G1tKl`(7G_xblMQANQ+j&NIeH&TK6-$u*4Uh&0t&ePU zPJkhRuh#-@_X+0}aV*Jb0Bfa+LZNqQVWJ0#=KA~Bqt%4}(36~^U)lvrj$CQX%P=?D ziHvZYaHPO6-Q>+|s~lNFW0?Bv%tzi)3M>X`;!RfF3<~0HjHc|}*l~bKATK4IXdR!B zMf+A}Up#I+)T8aogDs8)j}J)JK!%rH9&J59H~Q@Ntd^EV{~c7kTX%dQB_?kfOR-tn zA=NR@abtm5k{N9NS^G$1>>Td<278}g(`E7_k5+?RgoT&-Nqa5AjkAAn7s8#Vc=*sd zmyzfjfeIp0Fehg1gbSQ(_~qXV=y0ShN7ck^V@6t(5C%IxDmYn-~2#bGniWG#vS zWlnC*Dbfin3QX!ZI-YRxCO7uBG+d>=s@*c0sPmByGDc2mN&24$GkoH0oitsFTV0_} z4iATfIz{jBODQY1t{lpUS%Q1Hzdel~82P1N#Cura_7k&{mUoI@q?W7&Jzo61$}3G7 zl`3shFi_Vnoh`5OIKHqV;wTULz2GkZgW0zNjk3t#5aH8tz(R^=;i?c~(3-;#WM50snq>qF)cu>}tWC*wTO7r93>;1Cbif%d{o% zC1Eyo7UwX41o7QLvdU_to(vzDD`*KK^3HBZvx@j@i1Nbt-w8Z5`>?)c;rXTjdt#k# zOfJED_)awGGGg*Z0Rgo!JN?rDkpZFr6pE4%K}BPXJ>0O@93hgvCGJz?oUweJQjnVi zNQKWhxNpSd36=ip(-D4iOtMG99MY(y86GtXS~1%=jipBb#D;tZpKmMRZ_t=10TL%p z21RJ%0X=&&WUDYBbTcwsof1(CDGDD)eW`d#Y*Z87@k z^{dy_GcUp~J?qJ=i#H#EeSsp^TSr@dt$%q>c3_o1F9sr_ta1PLWYBdi1BNUNu0`v` zvgB;K@#gLmv#tD2Mf21LHU0Hq2~Ro}Upex$#h~)93nAvxcS6wkM&UVy#4RnSG6QX9 zQ;r$p=AKnBnUe=hZPH*u-Q4Ta4COuQ7TQGIqbUi4&eot$D2GHljdSdbc-MK-t1R86opRwDuUN+ zw(1^ybD7grBO>ySm29}i&+s{~7uz?*?K;N9?Yw~zd6 z*Xfoqv-*O~(QBAVpOqwZ``Qmd5qbL#d`>U7rT&?h?FN=iYu*vFfck~?6h=b48;n}$ zQrzUxWJ{eaR2!*MSX=+F*)ECE#91?SmduzuZwQ! z!ydL4;ljZ(9R_<=q z!=`&+*DUw>CsM8xVDT-;zFYUu%hn$rxPXhKztEb98>7ow#=fdMWJ!i$jJ=MIBspC; zvoJ2R96iz*(%23uM#WtAe661ynV`4t?K~eV&7!-r+tg^aw3Jiql zX^)V(pEN2WfQOL4!JgVGIoQ~a8}Gy_4l92Wst~iEI zANmgs#tUnQcv2E7>g!{jjC+X-g)LH8&8VQNoBvicmuID9WQoa^S-h?S(POL5f({Fs zWfe|-nRh@hz|Ck@iKm0C75R&`CWwUy<05TSN_IH3aMaO_Kw>0#Pv&-Dfl7b}3qfofON-WA!AB)QpF2FTnvu;s>T;lA1&Fh0 zBl$6%ODbhP1gIh2T%!8 zZ%&Q`_{;znmFQruzy3PWP@echTsS*JR65#1s^Yda=tWMNX?a%+u|@dSu2I$CfK@Jn zawQv>0i4QnlbtbIr{`+ihYt_GdJHR=O@6{5LHt~olXhcS{M}I*a8tl}U4uzgBx*jp zRji6=dfc!=jHsx4K9~%u9#`zIn~cO6$jl}Nco#8;2pDgqvpvO#S|Y1K4rie3vqVCS zI#QhtFED4h{9VA1j=@RcVQaORXzjNxK8$SAK4wPeIC%aePdZXEx8yE+0I;$3%avkwY+41*ee; z&@xvi6UvJOhfU)RKMMK5Ge)~VT{PNe>z_T^X7?!+cO%0O9;nBI39kOtN@7LUz)ZmX zVkxf)8QPZBxVNXV%s6vVeKr}hCJ=hY`pM{cihwK~6q{=~trr;R=dFS{Nx9;4Zr!`7 zG7^c|#x2=Z`)Um#l$|b#-4ZUow`yGvfCXce%qd#AG~sxuJ6eX@lQ?Gjjp4vuTv(to zGf_0z8b@Z3BzdaEB6`wXLwFwkyA*4$k{>ml#wj!^5x4DqDUFA|FW+@VD-FJyK3ynY z+{Gi9YbWOrqc_u1`$TYn+)Y1`=FhpVDRPdVzJ(>N;7R=OCBBghMVep-7atEDV6AsR zbPurLbCNf;oXDMCcEh;jgbeA|IE5ZbQ52ds%s}TJ-6?8~*qMF3@X8c=bL@w}r$Eeo zYUC@E6+viob;vjUn;z&lgCas{XLW zcxyK?xbJRX+WU9|%5bsaPbm!Tu)E}a&!br8FTR3?Cb%vZ7|$~!=Ixn55uZS#3NRZZ zs<82Gtkto2fzIEbE1T5-++IkANc74_ zARU;|ap|KEBu3}J?H?y>a845^ydr)R0F1K65>38_s0!GY|0t(o^g;aU(_1BuV33!b zi%`3stu>SZm%sRQ;lF#YPI4YIjsAv*0wm?LyvmEf2gKw__$W9yX+jR-P0o&>kaw+` zGf&tUrybKn0W_!YI0F{}d-V@ih~H2E^+PAzPlxaLf!!ly_BXZb`x{oX?}Ft-Yf}M7 zL{95Z!O*@rVV2j3Pjafo*D)wz$d3nQ2r{c~F-B4MlK60ouc3wU3}PEHhb{(moORi; zz5Hl)0M*Q# zOMmV8+5Oqz@+KiFk}x13`>Sg5)om(PI7B*n7hy<%)eZ%l1W=X?1Jtm2HUs`O#YFrj z9oFV(XD8)A{GK75(qMrd3jxUxPO`+Y7MVo#OtQX}E3fEqAVqj*?6JOOe$$5fn+5s? zx6moNC@o%1rwax68*VH@V-ANJ;x0GK{o3~V@1MKuiCN^IycAo;ZVc_;2O7q6eCH1I zoe1{_eg#}yXybiKf2$)I+FsNMa7IrsH~HZ|$A{s0LJf%{UQD;+jsdG?0>7hBQV)4Z z9Aj3a;Zp^Un5Ljqh`L5U{X*^*a6hqP--eRfh0}0|6M_IUiNtOni5Fk^t?onDM*MD^ zJegBUHkuv4>|8kN#xJYTzk`=4HR0PzpzJwG>KT()`#P3VF~fM5zGtG$RvQ|WmyaWj zqa&<4PU$5f921)o=e5(&Jm@$x-k);(lbnuD;XVQ&-lY< z+qf+FM4LeIsrObq4%f816^m|}8*00qF5^nxMS|H$dd#|s?}S(ciSghkJ(SJ=5y+twusP{MwkwIq zG2jBiouA4dgIuopX4Fp~UOni({ADA{&bB1_SYl{Q1wI*BTif%ee(N*7Z#OJCY z`He1l4dzecQ4W@TWAOkMgb_`GjENXd#_HoZ02Mr-Do>Xl9w;r*JD0R$si9tO6>US| zW|-ViVwqmhC1e{PTM51QN-HWn*EaOG$)PA8f8Q$HRNa&V^1`9Dp(-VE<`-cJRki~l zeQ) zV@HnYenHV4B4{V-j?tY(Fc2FsQ|x6Gw;Our*EHIetWC6h>UX4AD|F*5bjP5T z@3kaY0O%|F3o`0WTWlQP;ddr(jcn4KyY(k|Jxi~yT38Bltin0O;H6rTSn6Vcdf`n& z3VU99zPfSZtoV`jNq@?f5~?~6My$>J%7mhCr9$Go0cVO)?rpbQDqH4OAWGC zt!B23yF^#B>^~P@O$qgThx4S#JI`u=3Vb8kfuoSrCVyU3+I_TDPtMd zh77hUa;@t9$3OrpW1;dq;7e|B=27+?L&)R206N7fz6u?Vpo*g6vIY5v1DKt|AK$2M zJi?{ZR|-bTbSdNw@;C%KmF)oF@02bTYv#S(-3CkWy`T4^;;km9dfr10T|IR>C-<0| zdFuPGMJ!X;7kkg1rSdU~d23f8Z6O>Wa7!Q!!DKWHYFT(lU)%HbfN|7|CApdi!p6M* zZmPd41(qS*oGsEeT8dw)S%!yhgr&Tky+y^toYWPz1+9)DO8jzecE{}r$;iVGY{|@p zrp?%)e$c+T^FP36!i|qrv2(?@HIV=2NN1;L5puOPYfUZcG0NMuFx0O6`UePVOQ79wGgMj)l5<4?a<`Yl_RhY_C7U=0zKBC2$EhP^_G|S) zwv*z48K19@_pT*WUhAAZmlp){uf+E+7CcPp@0fe!wZ0R-R5-^z@HriduQz zZow5@W~ILN%8FlEM2p$(xE>5I81*!?MyluZ_h+)_1Ug0r&e(>Yv0M~3hqW5MAzFyu zT~rkx=9&{Z2Vck0$yI7kx_X*?*}kLE$UCA?X#yX}J5mqJIW0vPm&dE7bya_O96Z%~ zl$ilJ>NzFyNQyi0rMf#i6p;Rs2}#%Va%#q3X3af9vR@Gu^|I*Uw9XEY{t`plKE}Dw z8XFLZIremOfC4J$_eo{BWTsF}V-fd#;9O9P@gDn1IpW}EqCsR)gC7BFD#!|v9*h%1 z*&6syZPLg3GRsaVn+HT0jx{p1-AFJ$!XJPR;zEERi4XWy8F%Ob0bCHy{|+cVgt zxUeBR@Fg+_?_9G>{k)>Pg*RYkst}Ve&Yr9ku!oPKAT5$zr_hh$bio?MkK~VXg<}A0 z(xHUlM(j$|fxDCvX(ON*g)b7>LKCWPKjS0%J1wRdl;<;+3;S1WAQF7)9UG>EBPO4+ z+60A8s;x%l0#{t#>M3qq-pVQOPavJPiz)V?3tAxyIwpNpQ#BQ7cUn49TfXdRMw84e znq4y_=;tRzm6)Uu*a@=Cyn@(7`XL|*GokZSuV40Fdtg?L=UjQd71V&Il|4)T&J8z^ zX>1PZv)eLcn%pp%s3)`~`Cg;oBWcd_nBp_R7 z(cbpAAxWQ&^ZmRDkLbO=Jfb(k(=z$y_Dzc|sd{p_6S+9#Fbr7HEPqyXNdaJ3`3u6( zWDF@;ybOj>Le%rvVTGL7*S;P6;T6lI#?Yp@KX&- zeXq*<7IsOCb=uS5s0Mmf25>+hk)wj?se_5MedT~~WtEfn%Dxk#_W?Lj?3>GwN46fK z!IYgVw^_>#<=3oy;69J;(4rMSQ*bk#e z*O9H2VyX^(Rhj_h2~RKjRb;#jfWoVR_7xu0|7d;#jJeOlwzc=%h&6f;S#I99}wvxDNo zQFoYVq&-Mp!>+&et%Z3e-=EL?u?LUtia5D*zj}rztU#KX9V6C7;j7Q8S0 zlB*6q%yF@-Yf+q;a1)&^0$8&K{HXDYS&Ed)vJ!l6r$n9U8P`MUQZI)eK-^u6*Kdpf zzNar-y5wx;ZtRJpbYCGEd0*84PVL8&+BWu$y*{?sk&bhCehjZArP1SSX2_6(z{nE6M^R*|f6 z$ynra_U-VwV*BF1^ho4}C9XiaVprNH`hGFmgiUX%Pv*@VcTI~^;m|JEntHi&{_L&; zNnO;cWA4aJODk4op9K>jC_D0@eyJFuB2hh`Cwo{)#83w{6&Ky2xe7(Qnzks)2SH`f z9MmfjA!;HpQ_Q@C+Q5Zs>7ASx!lG`27XazRsQ1uR^eWQATS z(PqV@o6r#!swbqh-w^cNgLo54+nw2GAw@~>UnR!SfLMDZrFXJ!$OoPmtDTp_b;9`K z6tL5XDPoLt$~OS+O>IkYa^+oW@Jfg_g4g+JCAzGU4dsZ-rcx~ZL}!pigv95Pq3LG} zPEIepL$%a4dNpm5R9%Wqxwu3dl8$7pq4pjr{XIuHbFK8kLrI(}DqKPN12YQ2t3qzdnN!ez3Fd zp@($04skG7>K4pGr(&g2KJoRf`ea1&(??Wp<%O(8*U+X0RR*C;2`Ok6Xl&E2*5VdI zwm9bdWnitI-|PHYdRgj21CFGr*CO^yY1 zJkS;V*|!ymL(H~{Vz-foW=m%#Bb9256n3?)QAHTMGkd{94WY{Y;*C_3_M$LA@*1`k zcOc;KRtbu3LZZcSJ$Y@4f9q(6`;*$pPvvNuPTT!YP)11=@3hLs*qSRmT&kfVB_E~J`wO&l5No9Hxys8+F-y1{*16v=L0gph z26scBjUWa-_NHH!@XYfp&9h5bno!vSYX-@^Wni0>qJlmngFgNZ=RDuIzHu6Ja}IZ- zz~}h(TRXn514hbq<};7Yp!(msmGT0$WLE$i%+~T+S)Z&w;Z3dPlWkfIw!BJ{{~Rcq z;&sxPHBu7o@hrM#E2pGw2J~6gLR;dze8@5(Xd~jE(gF~%!U~&-tl;CBXIrbO$!#%# z7Wnm3NH%VXo`JPuS>tD|@@o51t zvF6hSTV`=L1picH03CEV53d&h8m~F=xI^xq$^KQg$S?s!Y>X4C8px}6>=*DKtGGqORX z>@+KMD)Z8^xQbawX$BD?6-3UNB<=xuVC8wB+3{ z$(6jJF;?=cj{Vw_x`S}-Rt)sM&?wC`WeCKUYuI|Su&3BBDm>S9B?@}*DAYqI@VH5J zx@#>WGMvy{SU5}Z-ds4VIzM&)$RV?;m6yYnO)4jn1+66*NN(r@8i51e)@X?XxljW& z!Mqh9S&j$#%jy30)1H zmLPP5mM-sO3a)B03I-**B$D}Mg=LNdyPsRNgzN$c%7l1~0s5sGk5LwCFlp`b1}{tY z`Ax$;Fh0h_WqU?!RsMi?(oU6P#~_3MRFz6_$2S%Y&}kOb(M&MiPm~{! zI`z;?7q`8^+qCNSK{t`or*wkUEAx){Js`RRh|P9E(`1{cvg-PRvg+x{^u&;j#m+6UDx{Mo^f1Zw);JI=wvFcnuMO()EMgA1m%4ZN)t=+tTUo{-mt26* z+YtnDP|`%#Mc4r*9=JNUppLb2m|;RLP_~8+D>BB^VX@~;nM(ASLh@oz5vUeD^CYnE z%sZ0<+!;U4eDkEZZ{0f~Z`$qI8Kw{pGxP)o=!I`)$0qyhKYNP`j1A-|^8Q z(IE~i2!?diQoAET^xIFq^XF(^gAzEOveZ#&@hY^0Wsx#jKD!&*f^7=zg?p!e4zYCx zm`g2=4;L3|Jv~$BIf>zyPp4%@okJzf`yPuSHMH7A&2cKN05YV1W^!P1%kc4LP+B=1 z_v)WD&+J|8+5u@+^?n)Tl-y?P6@xH|G0q5VL4U@?0e!W-O=L>!?VrBX+I?s$~ z+R^j|7)h>Gl(Pq9{aK<-m@9xaP!=*m9OgP;S(LE4#j`zVvSzF=uH6#r*@8;YNf6h? zM?C0=;hrzuLP9<(sJ`tcn#1=oI}cKoBNT{G4h~EsKbQ$)+upOKO24nXjex~C@DYjI z^H-KT^YiY_{qyYHG3Y~NID^UJ%(tUUUwxScD9C&CqBy=;?RY2TQ!LL8zEHK#JA-4h zjyvrS%@N-z=x&oyw-C1sVCr+(u(?A&MbAjX;!_=O(G+RJ=S%0kDY{G5j7R%f*!3Lu z4g14hdT%|ONka2%Mt^)pzcR6H!Ci>hDIGNc zI{I>=8v><;f>XvXd#l3P8Sj{536jWYa>{EhzwaYB%d0E%34 zs;&Z4pI+PJX=`lcUrsKkWLbX_E%z}twRY>ZWZ*ayyQpMM6JFI513Q{C3N3tqjZF3}4n~f@ z1^DS=&vW?GO_0n2{*g|QW&^Pcv|^Nh{_vAra`IX=Q)i-TJ>vbBs9PT;-Zf8d37A(w z!a&fT*gXFS6Cl`Ms(4TK0AUu%bg;1yNP>Qg`Kw6&A z+==jRb-{oPy?$sWM+5q(TH6-Hfq2}yOJs1A)gEt5iq_r(A0M%haJb?CJEE%{9MDb_ z?k8%7DL9hlwp;KtwOhovV+jatf2)5LG6%b3u;fgv&Cg)q9kg70Pa;_(Dp@-f085&lb{lrqjJ8XBwmAHz2ZU?>J&&Qt_utVGrOC;QXfP8-` z4(gvV_VMBckHXq0&CBQV*-Eb~g%i_xDBsc{u4VJ4V# z)zc`WeInwd{2}6{tnH<*T%#<~5YXqUVk1X0kyKV;V?B|?2qvfZWWJ%1d`v`{qzb8V z0%GqJ)!KpL8n(^YXvhTEPbM&N*Par2=zIcS*g*o-ew6NnE^4gHYxS2%ry#CtVr*@z zwt5j^SX@|L!FP+QdTwr(_G}*BfVwZnBq>D@EX6A;D}&V7K($g}Tv*OMQeQ4@(&KM| z2s5;`v-L$^DpBPqp^j)l1@*YY?SXH7bfVx?iP_RDr0jm5SQh>h;Fr&o!O%Lp_!MyQ(3)9E>d8DS=Y4e zX)UA3i+h_{j7JFweESq*VAY`P6_?Kr-?5{BV5qBo;43bLHH`A=dgd&kl&zpM)0G~- zkYP(@b$G@?HAcPDoRnK_YmTf}Ws}xe`c;l-nL+x$=@8O8&cTz-?T`>Xcq?7!eD(4w3I*^4gr*Mix$f6~Eu zL$d6&d$SyJiHzaTS(jn`-^OdoV(+^g%*5}4xiC2Aak%H8E}-9`mywb6OE#R#DUKP0 zdVGquO}fc|BHvLQwJS8k9BrC71m+*>?CBUI*L5bKEk5sD9UG+hR$T?L*a!IL8`Y<} z&x+sOGNWy`IELU&chBa@Wn5*JQwk!Xhw9c?0vrmnKecLQ>fuH_$bg-=YRIa%TxyLo zrXGl{;J`Zv|A^Xvbl*h*J0&R$R$Rl=v^#;vag}wz+Rgq4TQ~~#9XPJ=@F5%1fwVd6 zwJpeIYBSy8SmYE>Y_|F5&zWOuclzUs*!*9kb2>WvSW?oMoqvilS#gEiSRGUE;I)7W z)|E64QMUT8l=6U7@`hl*Ovr9SK?>h|yCXrQs?Za{(SF-2A^8r&;ma$yVXAv`?iY{Ruo_RpDc?$_mYe{$)!^{E%qV{M2lfi_`V{uh1LEo>ktW3KNwUB-O7WqdeNMZ^^ls8k6M-)JZs71vu_ddp;A!#g zw=wtYZZm1OVjZP72UQC)kLNf_2zE52^+~SYDd|&iCX;n0jA1Nw6}NY_8G`LN)DBhy zlWWng+oB7p6uXX_xHm4%EQ_n-YYtYEm)n7Ire#_8@fetEqAR^npHzl3SwWn01Ob3= z!A_Q3z;1)Bo}q*_D{yf z0m3N7l%x{&a?jd;^375PLG6R;IOpFh&DIHCqCl1a+`{_Se9*!4zMNmwTXL?t-{>jE z$Xie}xGj0iG^@ABlUF;!?(uq#xzp6Mx6Ul| z3hNeNoe5K6q?JwT%srU~F1bBLqFO8mC)Wd7Dz-`Q%l1u3F$h{!@}CpLAq!dM@jwH~ zzHhAgn;pmsF?>(7CxarmhWJxMrq1YZGA3Wz1@87!l!Y$CN7tfF!$-OzeglAe#;Fqa zb|lGe83*!xm~EW<$fAy1pN?N+1jh^7N;Fv(sOA#NdztDyHWHT705>9F7bCiiL`lba zuDrfhCqn3b@|o;We}3e5IwV1`^#tA^5N0csa*5^|Uaps2XI>j8J}+D#EV;>^A;+$G z{+Fs8c|#Tpo@yv3lRlyn4l|&^Jq!=;RL~3`^STI9=)eF$xiBRN8|}78od%veM~uY) z0C)8CXU0XqVAmNhW(c_;_7qO7P9Tn+s_`f9{trxKU`5_w6P2pjL)u0+J>yQ3gVFf0 zp=6XES5&pbv1@k6pqhcrgVuVtUW~TY!ys3EARHo4$Ke6b!DtC%RRM6oORchPV{wJY zZ}*hbvZAiz_e>FnKS<7#U`cJvJ>LqprgBT)h+^0Ho6q_}){b232RhdecEVytoPMp0 zb}X+S_}3#I8U0T`m*iv^+k>vWbCBpy_!MNYRb=0pTRjiRFc832V;`7x*oAZ;SCur1 z_GrOqO9Zi1Ne1W4*j)f`>&H2fMn&F+oRYW*b=kx34~c^V9_qgv*6_HFZ~iiEJits& zJgk4!dkVNb_Yt7=p~7YNNtUeMg9d6_pr;P4dJhBf@Gx$7RFGT^gE5s7moU@iGu znT^V@qS_zWer=95u@i1Gc?UB|gCk{NS3gMhr#ad8(I`@qG)aZ|UUS{}148nldRpo!`)^i0VQ@Qq^g+rJ?5f==gq7w{|_pWO}2l;^b=O{q0k^lGSE1USIAOou2v4CCA|EEaC9V5YiIo|(O)%OZ;|4x|Tf4Ktx n;|ctiLEZX40|KDl3KEuzJmfzPJO~KSzcU9N1Z4a0|3?28SkL|f delta 14892 zcmZ9z1yJQo8#Rc#yE_c-?(Q(S!{F}j7k6iHcbDPfHu&J~?p)lRft~-Y-P-*&ovJ=b zPCcEZ(n&v^a}uv1KMo-qHSCbPyRfYTA;G}#V8Fm=QcdiL0D3mg>h?Cy%x3l`Zf@Zk z3SJA+Sf4aal*3xyaB2f3RRkn*SV?+h;Z&T^;?_1w-kD)ErLoZ*yb=~;X(Oel*}4?iD#$8Yf!k8VzF5ri5)v$q$PmQzX#Mo_b>H9f*}wI2bh=zdc02i z;^4S!nnA%cfQQqR@Co07R@RcgmP`h7cPDz8z?<;!8ogf2z0PnSL>@*)EN9FgD7y@s z^W_ap{$|BPvj8b+wJA2d1I!7ej#qC9)(e&~Sw?Q#a|)ln6^VJ?vi5;Ni+ououb+G^ zbm|dvYPlMrwgWuk=$t>1Ao1yvB?XbREP9B>-xvpj0Y61>sF)?`*NhIiIs+}cAHqbA z#70YORkWhxs)3kJHE`d?Kk|%P`D&hpDy-YSd=k`&l|TIr>W@?Z zL7A=7dW%+}=x=8RUBgWhY%o=)t?9h8a`vU_2*AxQzi`Q2Y&Xrknv0Mr<8iwXf)>)3 z<**xfFVfQ9Sj^S9l~kQrqzQej1}+|6<=p28(#4VzP*g|RLouQ|xL>)e?aY5C>-_7U9h9=6~`#trpq4ttaDv%2@Bl~{dtJGpZ!6iID=J3 z37~>*=BRr#3KFW2AQdid5m84OEL(CEP>E7qhjqrN;Lp%DwroXr!VM6>`@|fHNuBr` z{t>g6<~8>PalEtbbZBC(`aFly>9EhKigz9(ES}BLoM_Q|0o6Y{>SY{Aqqc4{Zr5*X zI`0OfN6X1}#y5Q7{PX6LhG+)g-ed;_2H^Dz0Bd=reHdru2l_+HFbl$Q#)))JFfVY0 z2mR(+8#b?wl@n0{x}?#FCITWSS^Ug%A)%Hfx4n<~VD+7|HDFIv$_ejs2eU?=a*N{T zbIheH;rgJ*?Y3!+jzB+&$C0PmaqFD$%TezQvT3GYTt)iTq zKjmqowDPDslv)ivU4X%#$N@K1ECF-hDp-2mrNhn?-^)4v+I>70b9f3qV+6V*@Ditv zb?`iIy7gXnom^~L%>eu%cA5N(D5IbCW+T{4M#9HV&8H(>#QsQilZqi^42@e5YqO&F zQ{n_Ho;R!ioIe(8K6g+`BsTc^Pq`94ZV7ENxc#v* zh8_@c;!6i4@7cb=K{P<|HTI$9Ix`Hlv{(c9KJ?5ivi$Cko0J%$i}krLp%;KdU&p4i z4Z0o?`Er31_N$*JS@>}w5(i-p%jdZe%tXWI4*>I$5;@K6-V~>|_&3QZ_v-F}*>vV@ z?v=^f!M_*r9pa9@de-xk@={dBQ9U5bsC2`~lsBm>jlTqW7o4HJsRrh87~-$faUFnl zja&?aygao`O(WNP8hDL`4V}xQh?C@#qwMHi2k(g~9LtKU^w(;q4wPS@!c-<6`?Hjc z0dpgIuOY91h3z8zosxE7X~rhZ@F7z_duOVZ4j2Jw!~^n@*Rc>X4@S9gqE8nIv&ICO z6hBj9OjKkV?_smM&Sbj}nbBGYD<6<}s)JfM!ZTHpPA2#RRJ&)X?e{) zsaJ?h!r5?}%q*t+iG5!WDiRlaNNO@wUF%HX<#?EP$b`BL4+#U|b$((L+gKw-^%k+o zemdq-`Ne!PEp&>Tu>;}L@i#@uIGVw!OYF&BWThXI93thPv}67vGrbVAeTc~dFi1e( z4(1{k?mCs^4QQ+&_(a{#rT{eCZE$nAc-IacUt9?my^(i_4~kBH&Y1LT@2F^H!=e-q zkj+wipZG3pNGbPh1LSa8G3Fi!1Z%%RO#cm>xaTldF4rrw)c~ZsNNkAZi%!mJ z&dOE#v(cX2Uu+cMjFxKjdHWL02{j_*or_hD6i*MyP^80napiFY|9~zp%j4gPXb(R^SuO z15FztfoYjWtwwZasY41y?<|FinhI;cFDDhf;L9mx-&rtGtk{ioh|zetBQM%YyCxZ3X>aQex*ifMvglV(FS&z3q(GUXhLL$HS;V=k%cV` z(NT{50gFjSd8OANbvr}{XhW^)u4KXjKcnVr##Sp{*rPks)5Zr-yOdJB)9Ccp_GfZUcyN0U9hImp{JVS8Yx8f6Q|Ck7G~m?W5yAoAnzr8^t` zK~AvPGzZzue5g$|Da;?}^wSfkZz<&+xLJ6|9&lf=4s9UgqgZWtLm#<`a`8efYc$jR zk)y(I`f4D>OSsCPZDpHHmWxo4S0$}*%ufBWWS$m>!_5GQS>zU4+SFi*q|#5)$UU6c z#Y35zp4!y0lO|O>Ap1rDUm$Be8%_poL5B6W5kcpwZM7FG~axmn>+LqRc_JB{A zHgs|13VDKZ+eT3WG44un=ElhbCE9E9>P@^g8!YC(!<1M?q~$D6zrp^uD@QhJylr8C zfd$clfsy~~$|V1ua3ny-SMQ{&6AceJJ{fBiE4{)K9ECB2Dh39edA}kAj7B#V&sd*1 z&Ge>;OC6%4X3f%aUH#Jha+$RSg!C|TaZBC)ypsO=Q}4=??#}0%k;9wF$@W?b+x+v} zd&|dU$BF-mz{y5N>dX3dfnRb|`rXW3RaoFjQ6lJ>WO9U!H5w3%J$;{)LrmfulLvia z>IE(|7K5h|evc??mKYggKxU~2F4P~6fD0c5>2=4+h80^RY0?lW@6)L>i8iPxR;Y2L zyT53k7Jx8wJ1ZzWHt61CZKnIARXVZu+l16GF@y+@Ee1l;`AHjiTRDPF5qBlKZNcD-0iG71$bXvso z%9wU8XfRVVRI~)qq_+nXKJ%nPDWD-N8sP`6=!Rymtc77w2G;i8p753S8k!dptzhL%(zsZfS9Q0-QPTKe$e+eS5>+3` zqgc&^Y9jSD4Ziw2M;GVB0YB{RKcy`ZgVN1(rGHGN<7__l%tR9-CtH$*_EaRVcd+7- zq~mpJneYG{$Ykt3;OkvZN}ELN1D1{7c__h@&rerZ=Q_&F-j9##MeVF$XV*Q?x*pe) zNJwgtGv|!G8}q9g=`a$qd{;MXBljc5Ggz5)Ha45eE9(6GWZa(9r|aW4y7V`41pGSN z+S*!MT41ts_yv|>GTWELn%gt03V&6Um37$p6?y>dI7BUmG@7ew+zhqd$QpZWgkGHC z7&tm4lKaK_Z{!@3LB^NH8rP`!Eq=vsqfzK}4yifDa{ZkWq}*u8nGW2=zl^CSH3Zq^ zZq5vz{d4o3-CXQRj|W%5i}A76^DOD89bqI|F5lpi?jZa78y!bVjCUt5wlq_@c=6|h z1Y!UK5gp$!ww8#AxG7vPiyIIkLM$nMz^VzRz>8siW%N?$*w^`Py5Zxnl5Dvrh}<+vFZv>ZLEKZM61 znA=^jf_H6OdpUq?II^raf|U3x8OOcE)sX;9GJh!Pbl0bNDr}8{^G`*6ud7v?hpfj` z@`2@WaP{kraJM_|a2CxM_HY&}TM@S4@2geyne(CmMXFr5VR$X{)_{kZ(LQ)vxkjI( z0`>3ga3t>&+CLB7m_t0sc%w9Ueua$2ozr5<+Wwv*l25*z8+B|EGOT+V?w55?U^NHG zZZY@*exrfWu@Yii6z@c3^*081sXpmKx!rFIn@QU5JG-P<+O2XHn+SzL-e#g3a#*jX zA-MEV3bT?`i*C0{qoMqX>_X}{55{MERLMan;f!Q=WPeK~+YVaHVx&<@ZYK+7gf|Ro zSj)0+E8>knKQTriVvovC*+!9k^TY>~=k2LaLe7wL1lq{=O}F!5@D%w-kdAm7vF6I# ztU4fDInuKQ^ns!yXh02hMtclcy=r^k>HO0Mv>E)B5cozpokC2;ztMjkGKw1iSY3R! zyd}b2`8nVl@5{K#Glx0uMiAJP5{Bsgre?>R*r;dcO%~E>8A-yC&SHo1Jhl&LsbrLK zm{=;pLM15opj~&<9n)R)#TJ#Dfdgt80PvpGq2)GZ@yB2ELOD03@a$JT0x7brT~( zAnYt*w8|r>_G6GF+aBl@EiH1B4E1w1gU0GD=*7lPV#jmKa^qySDD%0+jdu68!kHV)wu* zR6Hl-u7WhPx~aEPw_+yIu4Yd({{qvix|hTG$+=T|%j91(Qn0s?S$+bbJt5ecZnOE& zeN#CQ7`jmYBqErj8=3`ay~Rnl&9xA0DYIJq#TrEvE|P;C{P2kvR`9ZR=h-Tp1G>Wr zbD3vTa#2z|Be>c6g}NH*BH?vEk_k#t{|%_34w#d{W!h-2VT_g%G;8UOzG=+KZ3sz!eQ~ygG=)) zT%Q=Evo8}L*zv#VBmTU?#}^z{aDEbyYP{IQ7wk3IeK781b7sj#=2aD%-BE`>T+f+( z7RoNpy+qkOtiYW`Vkuh-jz@9{56rM7510{%%s9v4hIyU<#H*zNhstr;Bi^i3W}Q@W z_@ZB;oa`4XFH*wv5gBOVpWwv&rw#Wx%Xy#dzwVI_=k|0ub}w^AC9>G+Z`;C70`!qs z5V46cf!aei^f0+EDBUhGMDe8=maT|fh+!Pu6>YK+AC^NR#WH3QKW0mR%r(qODR|Al zaD6f_d@|W}^6LozmS6o$#hV_twsJn$58i?5y&@qr+YOOL51Dh3F#QG7XCbmp)o(7N zzmTq}q^VvZ=3= z@!L11xFzPe*9n}Fvm?L}zIy!5K>>xpk*sf>oq7*wO#Ntx8nmq9f&fGSFa6%2Zvt_S zOU>abG@r6(XZ4$EIm{8IdSVOCf~MIS#@ABWdcqZucU5F^*vD=vqFBl@UYox*F&T2?sE_)xkp3FI&R!yngE?oVegg-Dzp zd*Mm7WYf`qE)6MMpIz0c4i4P#`4a`o)=pOv=EqOD|BMGT$z*^`i9^K^V_h3lQ(xB9 zy(9tZ4$L|f@Z~}_11xufY=g~Rh(k)!=b7Q(u9L0`Wx$(rTX}7wA2=q2x@$!6!fVTZQBG?g>`Xy$nKNu-=yKs( zHygJ-npfA8B>GB}f$Rdk$MO4WW-x>}`cP#J3s!XWbL%S7!Pyz6Z^v4l#$TupA~66b zI)J&BZ`gBqu|7quLQV*y^oA{)NyNpu>+H5C}aRx7EQVnp{ z>8+Pm9_4cT;D7k?RCK)*=tgW{s!x`A*yeVsEkGlAq{E*9jLPf2YTb;vCewwCF_;!?~_F zj#y&cdU^jL2UCO(gkM5O(z0tH03ea6YX1I$GBs{O_YkImG*gjabqd1W{)C2+G!}EzMTwUoOezvH| zmI(3@ll&>VK#pt){tAp0ngH*msdJfCLo$T6Yi9y#Yrf|SYme=lZr~&!>2vm9*p)FN zJbnQ4*8z+k;+9`fXAcJKmYBK7m+k7rdv40#>VJ`~sF{v=kau#N2 zMp{qNK||@X8HyW2t*))ItW+;M#nwi?x{R(Wy}VSI|r79A-N{?=nPMZu*9baTTuQUH5DMjq?K&GXOOJ`PG3SY)+^Px zY5C=H`qRe^QP%ssvTmNlRfncZewGfN-$Nl>W!vVo638r!nlK;xy8QFRQvaQm_*dOC zQT*QFeF~mB-aT&05RqRI{B7ipTYKoaL0Y7ZSP0H?#~*9eYdoea=)ERY`sd9enjIUlGcW5Zlz$g@9=&rYg6zpL6%NdGuNe8Gd)#SceU? z4;}utA=4nk{DNmPL+8wNYS5%#rE^^Rv#)mC{CG(jG{^n(IRk<`;!#`UzgKJ?S1#b> zZ>h-y@N3%7CLs);0YS{sliIipTBdSaX-RmAjRPPeR)Z3^6Ipke(1@i0Ay$F$G# zT!I#60qDdPsMhf>cmCGzkit@dOkVA{fy(aW4}s|ZO0Zg_QzhW$Ddg4S@w)N?$!VVC zz5t1vXOpvtver4c%fi^ba8=`BYo083>S0y8rvczIISNbJw^MfS^P>lcH!RR~ML{8Z zPvZDPTi+Wr{XDEYSAgtFQ0iX;u@x64!UoEq!O!jI;#?i93&=)X-9F6dv@? z19vPwE$Ab}Q^KfBe`kzxC(~nakuH#aAwUPLJ_2Mhi9r6x3k|WM?~ib)o-a0o)Qjdk zB^yu(gJXj7z8(Dapz9C})xN;PMJOP#7Zn-%R?RnWI|vZN%BKu{K&Dx#5-sk4K&%Z? z3g1=(IfQQ~XSqeKM$3}Q&?<%xW1Kh7yRbGK4oQ%cM8@gnm^=Lvx0A+t>*vML0Jtzi zy_2f2#z~AOmL#JmR=)%^6Qx(nxi zQ-6jmd?Z_ZN8|Mgvn+~wQ?=JFnJxEAi_jpjlP&uN^F~KRg<7FKKV$BT>o1}Ey97eV zQ(C@YBKSf0@84Th9}prj`wO}YVd>=hl$7;cy!aK`azMsW?(_|(O8a3?mf}nH z3yLH>f`QJ7=#Y3m9$oY|78@E#0f00~47qn@b@_an z(;cKui-(z}*W5^|N3n4)6%UbOn40r}W2dAx#sa!ue%S(4HC?H-tz$>|_F_-vP{|Vk zV-|Vp^(=CAhOPlNwwF&vTD9^r{UdRr4Sfappztne-z{P7LhaiQ$R1mZ!nRezaIq>B zqVfsU@@z1MY@I07apAC0#48=~}&cWqTPT5bE`GNbS%`Z*cQUYku zPN}rkg5{gn8e>Zd_B-mNLAw>--*1*zrfHwCpBvovOuZBoWs)`#n;7k^B~vbQPSksX zZ=`&mEc969(0qFXFOdogw=nGp%p#~eHNi#wb|fArU*P}d$AIJ+XPC$*HoRg>_+Vh? zTwq{i|E9)pfXp>J$bc15+m3llUbGa1c1o(1bm$a=l*h)j%}q#L-HeA`PO_0rie>XN z^7E!Uog3FnNi1#~?lhHe=%$PShU+TZz}-E&Vh0-qjyY7oV*vWtqEgjHtYf z&R)rcO7l?{D7|sau1cCoFTwqL3Jea1+#Fxw_$E+OYk;GMvVfWRq)$AbaR!o-?z{0n zqxwdVct@lv0{$eI8m=XV326#86nQWtTCgdbEo}y(s&q2Il5W|GuawhgF z%Ji*EX70)PA`B>&**su(cYthaT}(esCqL)|rc855MSqY;J3jJ7+L+c&{F=NpDi3{? z^BYs&-&W{!BjqEW5TwrUQL&Laf>UB{ASj|cYU;zI`2h%@;SyJ$V3_4Yu6b59tE-Uo z+K~wtUICgLlThWUp1U%;{U}LH2Ne{mqby8L4|3MHg?&f?BW+Mx18 z_IuqP#vyk-i0aCKHvCi=m(3E)#bAX?QbuPZ)-118iSkti^dJh5Nzim59G5EAIdlJb zY*m`6JAirkmu-@-HLT@zDcWVRkUL#KCbN3>B{Y`^*ejBd0!b}zXnsk<0kWQ)&AV2a zl$KL^>yeWCg^H6Y;y2!|nID|rIx|` zq#Ak}>5JzddM76ISG7dtu6_tc3{B-45akfcc(1IQ!D=2AI&GF=IE$SDS0;KoH4|pZ z-*F6=}ZX zP6B-3OXG{vDxgF3`Zn)AYj&fx7j#vweLGQVyv+W_>i`KE9K*7njhB>IZ>QXO0^kx{ zV%a?fkOVTg87TRG`LYG*cgTSK+O>E?LGr}Uz2ftgk_!2z2If8B$>W1bYpvrJ)r&}v zVzGKu8gFW5h<_Je%EaWR6;1t{2SI?3BN9-i9rqgW7ECN{1jV-YWN>8N@(#*vRUEEs z_CIp}wMNgG_VoU12?;GXnV^>6RTO>~hSH;z-wGl_l2mHP5Yz+N{uggx-)LRZYaZv# zo1WHp4|iq`6?=U~iSB6gr*>|QznFUUC}o{)Mdz2X90t$>&o?d5{LhtBNE}qB#}NPy z*{W5Gq}aE-wOS&Kz@LR_PysU3$c4L+z+p8vKV2(nz1d<11cY4_K7|9IuKS@wU59e) ze78&T$xe1i8JLtFeffouxJynw$xjV&M+tHD9aORVVg=$-6B20~Cj7oGus_gn`Viap z)BJboiUVY?sZ|;CZF5X>h30C0D-GbtCWUZ%J%w&Z?^op!FP)h$Ls6V%B%@JekO8?} z^=y8RlqXP;S0=nVz&j8p^Nq+m0FC4pjrEh&L1F}n%&Oc?Ut4~g`7O<%n^~ZAN^JeL z1;K`*A`&gX6}%ch`46Snl;>HyKD1zQPK+Lkn%#tn?YShg(axEUrjF>3r$qq2mGyH{ zgPLNi$x>XG%$Mq(8^0ye0^hqd0P(Q(nzCe>nnid8J!)~zlA##qbVPH%+IK&&nyz%N z8e?Uj0cBpA0nEX5Tj5pMsz1bJy?glNXFZ>Oy~}OyT!wkc{9j{72)sJYBGWQoJ=^uT zfv`e29xPVysxGuKKZIOgm`#8;GnNVrHly^D0SeyYz7I`4a^JIF6aa<&nEP-t@GvSC zeJL`DR5+;j9Lz%X(x=a#eDPUe$OpDkxnyU7v@kyqDoq3;%5fcT9WYSY_et}{@slyo zoA__|C&I9DAp^+i!Rw|MXYHI+=e#eU;k4iZP)ISNBl|`R*QIgzk^xZulD_Z`1u12B z!W2RCm4WT>Plb#fQ}}d8H>YN?Y?rp#?+`*G4oEiK3AuDK?Ym>fPJ0L|=jA1gCxkXX zk~wT7Cf}>{Y=;&-6AK;kN}kxIN5194o`zVl*}SW!nv*q(9A#8gGd^O3eR2;4;KM&- zlihXQ6p)f3e4#}Jqybt78Km+Q7*W(^FI$Avw?830Yzv$6wj&bx8$EG)O8ogQ>)4;% z2!}C8Z@FLh>eSOLV}89D()PQqWc*4Fi;bwZ8uJ00UJ18Va$fAw?j7EU@pY%xmXfJZ z-*=FysHrYlxO9ujZDFRfppwe>{U@Yxg;E&!RQ5$a{88cmvIdZR(S+Y+!|uz3g=Fb> zgPzP`z93MWr+BL3&%*l1S1Xf-tPb`Q6Dd$OLv~WGeQJ_OBk&yc=uyHnepLicpa!=B zO+yecFEQk)sF1r}OND+f z_dl$LF@jH>w69IA0i0VDelSLec6+kgNDFE6x1X)mR-*-3T*689khQfgVDmog{^DJve6UL2 zpfOM8K1XHARbU6)dj|++GHrZ7u5GY<#snaz{vA-^eADde6mfEOf^mdG{Q$??z0&H7 z>0^A&bc#XnHNcMy62wo-NYEoi%Ze6`_Me`VldMrKuU$C3a|tXoK^ST=JzQIr?5=MI zRfoDio}6ZzbhefigF*-0^N3{YfZ5vRH-cC<7V>X$%NRLMkb3#mn>wkaYYqe7#kJra zJOJ3^88~|`0d_|moIAg4rK#_>E?mRA#_?mp1b=c*UHG`vV>30d**CDcJ5KY3Qn!$D^yrsscj?Ipds93(`n$^ooqcrMHbC}4R^e~s* z@oN(QQoH7L?Us<@fA<;5AuAsHN;m%VvjVWl7im3Xvc45R`D_`)+v=h;Q0E&N)huiR44j%A9>2%J}tu^aE0C(5GJfwlc7CUD&YSH z7og~Gb}dX085-HWxBJWK0p-HG0t>_EZht}|{2Xf9Z@B#>w%Uqh+E;te2iveDe;V*$ zlk&YnP&kyvS?JZ93vDB6P!=<<->x!xrnsd$q16@f(UnlpR0zewfivoad0RBYRY0&b zw0_{;SJ3G&z6w&B&f|ti82U{&A&Lig+=%V4}>fRsih>I9rCuC~c8#CLutITP?(|K!XI#F^&^Q!n$&r<`H5kgFIH)fL4j^lqC% zDGfR6vE!rJregSe;df&_J&+{%iWc~mBgo*mJ9b1{i%%Xc;%c4e?OV_<;$SPMPBhIj z9w%}hr!w(v>4jJSp}&aM%uX}1=Vf%!3gGj<8KM<@*f=R|0@AB7Zh>5z3Eth0X6V7hwjBSz*NeBs(mee4F;T#Wh^5{VBx(@>%50I0zG0< z?Ge8|>d9J53NBU6VQmrdsN539WKQv!lImkfwTJHRQQDJ5Fm7S$M2JT5NPZ2NxI&zs zz*Bpf@WJN0ZqZ2I`i#SM#VuhLecRH(5W}(aE|@lioo}*a-51G;R_>4cPf{Sx@DmyW zZg7S!&OddG3S6p6C4MT)G7-Q~eL)l}Vn*C%9RuX`iiM7~UMMN10vW#u*N5+v z`Evxr9+O7SVr1tqe0tSo1Q8Gv94+D- zgdlPskSuN>0xSo7wRqx$)7)kiXBT=(fb(KL36qRPG&o3SfpKH8nhBuK;SNz!=5_?6 zIIm_RO^eNeqR4wR99DxL+RTqAUO7Toe&FADR{k{uM3_!~&B{3gVMVY2|`3xZnLaGl<1%Q3Z?Hrn7U$R!j3_EeY zh@o7%phu}7pj;P>T#ij8&uffc$p&odBoLdA~JY!NX3VK1=>$E-Ts;5ku zZp6iCT`jln?22p}!Do05z|{8K^1^NNo*Hv^VwqX*5nUeKBDV4sC}(wiWC~Y#+_RM? zuetB9Ydz^p!4MA0rFFg$l0uh3&c%Y{B-A|3`ODJ469JpA?1LVh;oj9PtiR)y?!(}i>(!_)`nF|-6$ z=H)stA;(hDEeJTa80sT}5pO^^;1t$$DKPG3_zOib470JDYWm3yH_g9W8>;5cHXpHf zoiM=^m%95W6O1$;UHl7c-cX(b}i%B@^N z(48q?hEh9s_zHZTiK#`byC0sf%dIlYi%88e<3v>Zp&9_{e>M(=+&2@$X(x+KIu3r( zL4)T~2oMF;g8K29qxwP^-NdMb|JAjHmMy5V1CYA=A#sgl=LSjd{z>RK=8#-D0ir1+ zqmaz9LC|BaV(G7B;5g>ETphw>bf}WYAyB$WLd>HQ!m>%wKJnQ+0iq*%l~ED{~uvln@+CJ20R#8EjAb!?f*%+ zQ+L*I0Y1i9N7!FVO*v~wsm9z?XmFjTKP|k-V^q=5j^He~w1M!P#yQH|spjTD;PkYs zb=|O*9qOqZ(^G5RB96X2c~QAMYD`_v^?UF2dwI)s0LR6&BaFh=>TAMt?@rgw^JVIn z&w~pX!>toOOY-eJno)Tn0!xNVLkJlPZPE<_VB4oGPCNX@7QaE&8P}+$5C;}}vL773 zL7f#B);9WH__I4-B=TkV?}rbh`VQVej<-L@b$7Ux6Y`#epm1M7TjUK2$(@zKdwc8eqGw!Ul?mCN02fgw_ z1sxrjMi+_dg-{jciw)MsB?$u+X+?)E0BiSMbxovt=oZHDwd@me1&r^z00X+vPxEO$rzdR_YR9ymou&{zu)K*!1TTRG9EJbU-s*MS=o_hC%b+vx%ubY~WHvf~kvu^k( z5pmgY2w27`=qy|49b6uyb7#+OJnQHsOt(0BjVOgw7~8a(Se~jJWZER><~%m{0M;5o zc6#qr?vfMz1t`DV8uFQE*&q<@*=6K_9fs0c*K~>rpyeR$fzF7o$>#L6a$T5)Ev43t zG=)!cA%nhN1c`IC*7WVAx}!}uuJgEBlZK4OW^o0;3eyISSh1N>zW?cF&azuQEW}fo zSb~#)2xg93dj0}q05G{CmynJXFj{CK+fLRwiJr7{`PBbO1xw|GQ|nHrK^>!}LB?{R zZeCnwR{}9l)XeTqW@cLwklzf4uRHEyn8Ua(CjAZA5prqYkalZ>UyyvO>-yF1=(j|< zWnIB|gRwvN^-aOt&^t(R4S$QT>*^yZ#UL^(j>VzGX1%l^{d{?qd8)|+pfE&NsC!`U zP?CtGHsDM~-7K6Z3V$!{e>0~>w|Hr z{igU10dQ2imGX}!2pl{96kq11c{C-Kmu=^llHW~cQ=@5mnE#j`t(2RnwUK$~(a>Y4 zESJ~mq1+tN@W=mQV)LVH+C9IlY(ER6Jr_@c-2+l*>+iJ1Q@!N^_~(Vi`JQ=~q_1fD zL+)s}FgR-8GNo&b%vG#m()Ugg?Ui`q@qrCczxDc%7!lF@K(wN=2eDBW(^L2% z`B5|}?3|R!2v=0Zvq_M~;KGvgIkqp?Oo{*XN<6g;PH?wten{#-W9 z_rNmg^|2;7o{))iC!W*!4!BmsBbye}a}YO# zcX;ps;ANN!1ZbY1~hv1vdNMKW4PuVRTmoAo2vMh?jDvQ6SwCzL6R=1Fh;lLRni zs4|%^F2D`JQwD3*-i*q(TV9}bt1%$EKMRPL5fQ`9PFJmRp22%Fga2?QLjE=65@vRL zU>%pr9eHCc=mK$X`X`D#zMPIT*2Y^HRb7V_5T8!R=>CMm=T~Ry^b6=!1oT4pp=A$` z&6}d0KBf-&HMQ2YxYnh3!Q}B&JiXmylVr6Y`KwW;-Lm5#o43pIl~XI%Kg>R6mz;<^ zmAJxQ3^JgB3~>X5`Y1m+n0EMvvfr7#-;0o8#&xvJg%!t@Iiz>-ho5MuCCo*rsP@kw zpgrL;)Cp@k4t;#kdIWe&w0EYCH{u4)W(KQZI+CSMZLk$rT>)2`9YS9sU;g`vlg2uO zl>Ol-Nk2?i%8Zb&r6*P};1x6X`%i^Gv%KL9)>hOI`u|k24S4iaxBXVs0{XMJYHH39iKO+wUILxLBh*iwb~6HP zr-J@!ayCPucsqKI`V0+_1SPgC-2tpu z20?po6xi5Ery?X5|1|Q@5Tf@m%DwmCehnz%HKbl&khnib{k#VcnGMy6MLCJzSB{mSru-M7YIf>C&TK{asy8rb%F zI0J2{ddgkg_P%$+U07>uEGhXiF>IfuY*B?>PFp<)8O#cFMIu9gxRzhM_L}3WRT{(! zvT|tI;t12!ldM-%E8S>_&bSt*Tav&3U>3F(GdoBbt{YJLcz(+}1Y;VCwPqn}(iVHf z53|_BuBEQ;iZwYadD~U5D^_qs=rnYt?Nd6s5K`OA@DnPsV>+8ZJEPbe4*AOef=KN@ zBm%x3kRkp5OocQz^sxW8sW27%1Sj>?1r6z+7vaC9G#Jh)buJJ)mB^JS74`%zRpOQa z95ogEmOeG=mKDOx^WQ;|)F2<&)SX*2qW>&VP+(xI|I7@513LtG>3`6<67&CD5z+tri~66YM#}#Y z6(QF8{)=7u$PE!b_#a#uLrxjR`|p0xJP|MOB diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ee9745cc621ae..f00d0c8442459 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=5625a0ae20fe000d9225d000b36909c7a0e0e8dda61c19b12da769add847c975 +distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640 diff --git a/gradlew b/gradlew index aeb74cbb43e39..fcb6fca147c0c 100755 --- a/gradlew +++ b/gradlew @@ -130,10 +130,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. diff --git a/libs/common/build.gradle b/libs/common/build.gradle index 003a04ad4bfa7..973fe30d09842 100644 --- a/libs/common/build.gradle +++ b/libs/common/build.gradle @@ -13,7 +13,9 @@ import org.opensearch.gradle.info.BuildParams apply plugin: 'opensearch.publish' -archivesBaseName = 'opensearch-common' +base { + archivesBaseName = 'opensearch-common' +} dependencies { // This dependency is used only by :libs:core for null-checking interop with other tools diff --git a/libs/core/build.gradle b/libs/core/build.gradle index 65d507c37b3ff..46b6f4471731f 100644 --- a/libs/core/build.gradle +++ b/libs/core/build.gradle @@ -32,7 +32,9 @@ import org.opensearch.gradle.info.BuildParams apply plugin: 'opensearch.publish' -archivesBaseName = 'opensearch-core' +base { + archivesBaseName = 'opensearch-core' +} // we want to keep the JDKs in our IDEs set to JDK 8 until minimum JDK is bumped to 11 so we do not include this source set in our IDEs if (!isEclipse) { diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 6c1785e6faf6b..3b3f2a7f9ca38 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -109,7 +109,7 @@ tasks.register("apiJavadoc", Javadoc) { source = sourceSets.main.allJava classpath = sourceSets.main.runtimeClasspath include '**/org/opensearch/painless/api/' - destinationDir = new File(docsDir, 'apiJavadoc') + destinationDir = new File(file(java.docsDir), 'apiJavadoc') } tasks.register("apiJavadocJar", Jar) { diff --git a/modules/lang-painless/spi/build.gradle b/modules/lang-painless/spi/build.gradle index 7811ee0e41dd8..59a77870b4987 100644 --- a/modules/lang-painless/spi/build.gradle +++ b/modules/lang-painless/spi/build.gradle @@ -31,8 +31,10 @@ apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' -group = 'org.opensearch.plugin' -archivesBaseName = 'opensearch-scripting-painless-spi' +base { + group = 'org.opensearch.plugin' + archivesBaseName = 'opensearch-scripting-painless-spi' +} dependencies { api project(":server") diff --git a/server/build.gradle b/server/build.gradle index ab67eabe76d0c..38bbf020d860b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -46,7 +46,9 @@ publishing { } } -archivesBaseName = 'opensearch' +base { + archivesBaseName = 'opensearch' +} sourceSets { main { diff --git a/settings.gradle b/settings.gradle index d908e941b3d6b..b7d47cd9b745e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.enterprise" version "3.11.1" + id "com.gradle.enterprise" version "3.13.3" } buildCache { From 2f9728e8398e923cbbb93e36dd52c22e02612a8d Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 6 Jul 2023 17:17:28 -0400 Subject: [PATCH 062/186] Create concept of persistent ThreadContext headers that are unstashable (#8291) * Add unstashable section of a threadcontext by adding ability to put and get durable headers and transient headers Signed-off-by: Craig Perkins * Add CHANGELOG entry Signed-off-by: Craig Perkins * Remove new lines from top of CHANGELOG Signed-off-by: Craig Perkins * Only one threadLocal Signed-off-by: Craig Perkins * Update CHANGELOG Signed-off-by: Craig Perkins * Single putPersistent and getPersistent Signed-off-by: Craig Perkins * Remove unnecessary line Signed-off-by: Craig Perkins * Chain putPersistent Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins Signed-off-by: Daniel (dB.) Doubrovkine Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + .../common/util/concurrent/ThreadContext.java | 74 ++++++++++++++++--- .../util/concurrent/ThreadContextTests.java | 29 ++++++++ 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab2ebcd9f3562..766a39db45781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) - Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) - Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) +- Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java index 025fb7a36b684..9dd5d21a00231 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java @@ -149,7 +149,7 @@ public StoredContext stashContext() { * Otherwise when context is stash, it should be empty. */ - ThreadContextStruct threadContextStruct = DEFAULT_CONTEXT; + ThreadContextStruct threadContextStruct = DEFAULT_CONTEXT.putPersistent(context.persistentHeaders); if (context.requestHeaders.containsKey(Task.X_OPAQUE_ID)) { threadContextStruct = threadContextStruct.putHeaders( @@ -262,6 +262,7 @@ public StoredContext newStoredContext(boolean preserveResponseHeaders, Collectio originalContext.requestHeaders, originalContext.responseHeaders, newTransientHeaders, + originalContext.persistentHeaders, originalContext.isSystemContext, originalContext.warningHeadersSize ); @@ -337,7 +338,7 @@ public void setHeaders(Tuple, Map>> head if (requestHeaders.isEmpty() && responseHeaders.isEmpty()) { struct = ThreadContextStruct.EMPTY; } else { - struct = new ThreadContextStruct(requestHeaders, responseHeaders, Collections.emptyMap(), false); + struct = new ThreadContextStruct(requestHeaders, responseHeaders, Collections.emptyMap(), Collections.emptyMap(), false); } threadLocal.set(struct); } @@ -375,6 +376,13 @@ public String getHeader(String key) { return value; } + /** + * Returns the persistent header for the given key or null if not present - persistent headers cannot be stashed + */ + public Object getPersistent(String key) { + return threadLocal.get().persistentHeaders.get(key); + } + /** * Returns all of the request headers from the thread's context.
* Be advised, headers might contain credentials. @@ -434,6 +442,20 @@ public void putHeader(Map header) { threadLocal.set(threadLocal.get().putHeaders(header)); } + /** + * Puts a persistent header into the context - persistent headers cannot be stashed + */ + public void putPersistent(String key, Object value) { + threadLocal.set(threadLocal.get().putPersistent(key, value)); + } + + /** + * Puts all of the given headers into this persistent context - persistent headers cannot be stashed + */ + public void putPersistent(Map persistentHeaders) { + threadLocal.set(threadLocal.get().putPersistent(persistentHeaders)); + } + /** * Puts a transient header object into this context */ @@ -566,12 +588,14 @@ private static final class ThreadContextStruct { Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), false ); private final Map requestHeaders; private final Map transientHeaders; private final Map> responseHeaders; + private final Map persistentHeaders; private final boolean isSystemContext; // saving current warning headers' size not to recalculate the size with every new warning header private final long warningHeadersSize; @@ -580,18 +604,20 @@ private ThreadContextStruct setSystemContext() { if (isSystemContext) { return this; } - return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, true); + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, persistentHeaders, true); } private ThreadContextStruct( Map requestHeaders, Map> responseHeaders, Map transientHeaders, + Map persistentHeaders, boolean isSystemContext ) { this.requestHeaders = requestHeaders; this.responseHeaders = responseHeaders; this.transientHeaders = transientHeaders; + this.persistentHeaders = persistentHeaders; this.isSystemContext = isSystemContext; this.warningHeadersSize = 0L; } @@ -600,12 +626,14 @@ private ThreadContextStruct( Map requestHeaders, Map> responseHeaders, Map transientHeaders, + Map persistentHeaders, boolean isSystemContext, long warningHeadersSize ) { this.requestHeaders = requestHeaders; this.responseHeaders = responseHeaders; this.transientHeaders = transientHeaders; + this.persistentHeaders = persistentHeaders; this.isSystemContext = isSystemContext; this.warningHeadersSize = warningHeadersSize; } @@ -614,13 +642,13 @@ private ThreadContextStruct( * This represents the default context and it should only ever be called by {@link #DEFAULT_CONTEXT}. */ private ThreadContextStruct() { - this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), false); + this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), false); } private ThreadContextStruct putRequest(String key, String value) { Map newRequestHeaders = new HashMap<>(this.requestHeaders); putSingleHeader(key, value, newRequestHeaders); - return new ThreadContextStruct(newRequestHeaders, responseHeaders, transientHeaders, isSystemContext); + return new ThreadContextStruct(newRequestHeaders, responseHeaders, transientHeaders, persistentHeaders, isSystemContext); } private static void putSingleHeader(String key, T value, Map newHeaders) { @@ -637,7 +665,25 @@ private ThreadContextStruct putHeaders(Map headers) { for (Map.Entry entry : headers.entrySet()) { putSingleHeader(entry.getKey(), entry.getValue(), newHeaders); } - return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders, isSystemContext); + return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders, persistentHeaders, isSystemContext); + } + } + + private ThreadContextStruct putPersistent(String key, Object value) { + Map newPersistentHeaders = new HashMap<>(this.persistentHeaders); + putSingleHeader(key, value, newPersistentHeaders); + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, newPersistentHeaders, isSystemContext); + } + + private ThreadContextStruct putPersistent(Map headers) { + if (headers.isEmpty()) { + return this; + } else { + final Map newPersistentHeaders = new HashMap<>(this.persistentHeaders); + for (Map.Entry entry : headers.entrySet()) { + putSingleHeader(entry.getKey(), entry.getValue(), newPersistentHeaders); + } + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, newPersistentHeaders, isSystemContext); } } @@ -658,7 +704,7 @@ private ThreadContextStruct putResponseHeaders(Map> headers) newResponseHeaders.put(key, entry.getValue()); } } - return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext); + return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, persistentHeaders, isSystemContext); } private ThreadContextStruct putResponse( @@ -695,6 +741,7 @@ private ThreadContextStruct putResponse( requestHeaders, responseHeaders, transientHeaders, + persistentHeaders, isSystemContext, newWarningHeaderSize ); @@ -730,7 +777,14 @@ private ThreadContextStruct putResponse( return this; } } - return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext, newWarningHeaderSize); + return new ThreadContextStruct( + requestHeaders, + newResponseHeaders, + transientHeaders, + persistentHeaders, + isSystemContext, + newWarningHeaderSize + ); } private ThreadContextStruct putTransient(Map values) { @@ -738,13 +792,13 @@ private ThreadContextStruct putTransient(Map values) { for (Map.Entry entry : values.entrySet()) { putSingleHeader(entry.getKey(), entry.getValue(), newTransient); } - return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient, isSystemContext); + return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient, persistentHeaders, isSystemContext); } private ThreadContextStruct putTransient(String key, Object value) { Map newTransient = new HashMap<>(this.transientHeaders); putSingleHeader(key, value, newTransient); - return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient, isSystemContext); + return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient, persistentHeaders, isSystemContext); } private ThreadContextStruct copyHeaders(Iterable> headers) { diff --git a/server/src/test/java/org/opensearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/opensearch/common/util/concurrent/ThreadContextTests.java index 64286e47b4966..dfa239757513e 100644 --- a/server/src/test/java/org/opensearch/common/util/concurrent/ThreadContextTests.java +++ b/server/src/test/java/org/opensearch/common/util/concurrent/ThreadContextTests.java @@ -71,6 +71,35 @@ public void testStashContext() { assertEquals("1", threadContext.getHeader("default")); } + public void testStashContextWithPersistentHeaders() { + Settings build = Settings.builder().put("request.headers.default", "1").build(); + ThreadContext threadContext = new ThreadContext(build); + threadContext.putHeader("foo", "bar"); + threadContext.putTransient("ctx.foo", 1); + threadContext.putPersistent("persistent_foo", "baz"); + threadContext.putPersistent("ctx.persistent_foo", 10); + assertEquals("bar", threadContext.getHeader("foo")); + assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo")); + assertEquals("1", threadContext.getHeader("default")); + try (ThreadContext.StoredContext ctx = threadContext.stashContext()) { + assertNull(threadContext.getHeader("foo")); + assertNull(threadContext.getTransient("ctx.foo")); + assertEquals("1", threadContext.getHeader("default")); + + assertEquals("baz", threadContext.getPersistent("persistent_foo")); + assertEquals(Integer.valueOf(10), threadContext.getPersistent("ctx.persistent_foo")); + assertNull(threadContext.getPersistent("default")); + } + + assertEquals("bar", threadContext.getHeader("foo")); + assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo")); + assertEquals("1", threadContext.getHeader("default")); + + assertEquals("baz", threadContext.getPersistent("persistent_foo")); + assertEquals(Integer.valueOf(10), threadContext.getPersistent("ctx.persistent_foo")); + assertNull(threadContext.getPersistent("default")); + } + public void testNewContextWithClearedTransients() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); threadContext.putTransient("foo", "bar"); From d333cd388e304459fd46999548571e7b01f576ad Mon Sep 17 00:00:00 2001 From: Mingshi Liu <113382730+mingshl@users.noreply.github.com> Date: Thu, 6 Jul 2023 19:37:23 -0700 Subject: [PATCH 063/186] Enable Partial Flat Object (#7997) Before this PR, flat_object can only be defined at the root field of JSON object in 2.7 and 2.8, because the json object is flatten so it lost the structure of a tree. Now, this PR enables json object to be partially mapped as flat_object field type while other fields in the JSON can be other field type. --------- Signed-off-by: Mingshi Liu --- CHANGELOG.md | 1 + .../test/index/100_partial_flat_object.yml | 606 +++++++++++++++++ .../index/105_partial_flat_object_nested.yml | 636 ++++++++++++++++++ .../index/mapper/FlatObjectFieldMapper.java | 63 +- .../mapper/FlatObjectFieldTypeTests.java | 134 ++++ 5 files changed, 1424 insertions(+), 16 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml create mode 100644 server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 766a39db45781..ffc9e6d2783ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) - Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) - Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) +- Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml new file mode 100644 index 0000000000000..91e4127da9c32 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml @@ -0,0 +1,606 @@ +--- +# The test setup includes: +# - Create flat_object mapping for test_partial_flat_object index +# - Index two example documents +# - Refresh the index so it is ready for search tests + +setup: + - do: + indices.create: + index: test_partial_flat_object + body: + mappings: + properties: + issue: + properties: + number: + type: "integer" + labels: + type: "flat_object" + - do: + index: + index: test_partial_flat_object + id: 1 + body: { + "issue": { + "number": 123, + "labels": { + "version": "2.2", + "backport": [ + "2.0", + "1.9" + ], + "category": { + "type": "API", + "level": "bug" + }, + "createdDate": "2023-01-01", + "comment": [ [ "Doe","Shipped" ],[ "John","Approved" ] ], + "views": 288, + "priority": 5.00 + } + } + } + + - do: + index: + index: test_partial_flat_object + id: 2 + body: { + "issue": { + "number": 456, + "labels": { + "author": "Liu", + "version": "2.1", + "backport": [ + "2.0", + "1.3" + ], + "category": { + "type": "API", + "level": "enhancement" + }, + "createdDate": "2023-02-01", + "comment": [ [ "Mike","LGTM" ],[ "John","Approved" ] ], + "views": 3333, + "priority": 1.50 + } + } + } + + - do: + index: + index: test_partial_flat_object + id: 3 + body: { + "issue": { + "number": 999, + "labels": [ { + "version": "1.1", + "backport": [ + "1.0", + "0.9" + ], + "category": { + "type": "Module", + "level": "feature" + } + } ] + } + } + + - do: + indices.refresh: + index: test_partial_flat_object +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: test_partial_flat_object + + +--- +# Verify that mappings under the catalog field did not expand +# and no dynamic fields were created. +"Mappings": + - skip: + version: " - 2.99.99" + reason: "flat_object is introduced in 3.0.0 in main branch" + + - do: + indices.get_mapping: + index: test_partial_flat_object + - is_true: test_partial_flat_object.mappings + - match: { test_partial_flat_object.mappings.properties.issue.properties.number.type: integer } + - match: { test_partial_flat_object.mappings.properties.issue.properties.labels.type: flat_object } + # https://github.com/opensearch-project/OpenSearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test#length + - length: { test_partial_flat_object.mappings.properties.issue.properties: 2 } + - length: { test_partial_flat_object.mappings.properties.issue.properties.labels: 1 } + + +--- +"Supported queries": + - skip: + version: " - 2.99.99" + reason: "flat_object is introduced in 3.0.0 in main branch" + + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 3 } + + # Match Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + match: { "issue.labels.version": "2.1" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.version: "2.1" } + + # Match Query without exact dot path. + - do: + search: + body: { + _source: true, + query: { + match: { issue.labels: "2.1" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.version: "2.1" } + + # Multi Match Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + multi_match: { + "query": "2.0", + "fields": [ "issue.labels.version", "issue.labels.backport" ] + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.backport: [ "2.0", "1.9" ] } + - match: { hits.hits.1._source.issue.labels.backport: [ "2.0", "1.3" ] } + + # Term Query1 with exact dot path for date + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.createdDate: "2023-01-01" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.createdDate: "2023-01-01" } + + # Term Query1 without exact dot path for date + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "2023-01-01" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.createdDate: "2023-01-01" } + + + # Term Query2 with dot path for string + - do: + search: + body: { + _source: true, + query: { + term: { "issue.labels.category.type": "API" } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.category.type: "API" } + - match: { hits.hits.1._source.issue.labels.category.type: "API" } + + # Term Query2 without exact dot path. + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "API" } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.category.type: "API" } + - match: { hits.hits.1._source.issue.labels.category.type: "API" } + + # Term Query3 with dot path for array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.backport: "1.9" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.backport: [ "2.0", "1.9" ] } + + # Term Query3 without dot path for array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "1.9" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.backport: [ "2.0", "1.9" ] } + + # Term Query4 with dot path for nested-array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.comment: "LGTM" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Term Query4 without dot path for nested-array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "LGTM" } + } + } + + # Term Query5 with dot path for array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.category.type: "Module" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.0.category.type: "Module" } + + # Term Query5 without dot path for array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "Module" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.0.category.type: "Module" } + + # Terms Query without dot path. + - do: + search: + body: { + _source: true, + query: { + terms: { issue.labels: [ "John","Mike" ] } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Terms Query with dot path. + - do: + search: + body: { + _source: true, + query: { + terms: { issue.labels.comment: [ "John","Mike" ] } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Prefix Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "prefix": { + "issue.labels.comment": { + "value": "Mi" + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Prefix Query without dot path. + - do: + search: + body: { + _source: true, + query: { + "prefix": { + "issue.labels": { + "value": "Mi" + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Range Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels.version": { + "gte": "2.1", + "lte": "3.0" + } + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.version: "2.2" } + - match: { hits.hits.1._source.issue.labels.version: "2.1" } + + # Range Query without dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels": { + "gte": "2.1", + "lte": "3.0" + } + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.version: "2.2" } + - match: { hits.hits.1._source.issue.labels.version: "2.1" } + + # Range Query with integer input with dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels.views": { + "gte": 3000, + "lte": 4000 + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.views: 3333 } + + # Range Query with integer input without dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels": { + "gte": 3000, + "lte": 4000 + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.views: 3333 } + + + # Range Query with double input with dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels.priority": { + "gte": 4.1234, + "lte": 5.1234 + } + } + } + } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.priority: 5.00 } + + # Range Query with double input without dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels": { + "gte": 4.1234, + "lte": 5.1234 + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.priority: 5.00 } + + + # Exists Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels.priority + } + } + } + + - length: { hits.hits: 2 } + + # Exists Query with nested dot path, use the flat_object_field_name.last_key + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels.type + } + } + } + + - length: { hits.hits: 3 } + + # Exists Query without dot path for the flat_object_field_name + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels + } + } + } + + - length: { hits.hits: 3 } + + # Exists Query2 with dot path for one hit + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels.author + } + } + } + + - length: { hits.hits: 1 } + + # Query_string Query without dot path. + - do: + search: + body: { + _source: true, + query: { + "query_string": { + "fields": [ "issue.labels" ], + "query": "Doe OR Mike" + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Query_string Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "query_string": { + "fields": [ "issue.labels.comment" ], + "query": "Doe OR Mike" + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Simple_query_string Query without full dot path. + - do: + search: + body: { + _source: true, + query: { + "simple_query_string": { + "query": "Doe", + "fields": [ "issue.labels" ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + + # Simple_query_string Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "simple_query_string": { + "query": "Doe", + "fields": [ "issue.labels.comment" ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml new file mode 100644 index 0000000000000..ce172c2773e1f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml @@ -0,0 +1,636 @@ +--- +# The test setup includes: +# - Create flat_object mapping for test_partial_flat_object_nested index +# - Index two example documents +# - Refresh the index so it is ready for search tests + +setup: + - do: + indices.create: + index: test_partial_flat_object_nested + body: + mappings: + properties: + issue: + type: "nested" + properties: + number: + type: "integer" + labels: + type: "flat_object" + - do: + index: + index: test_partial_flat_object_nested + id: 1 + body: { + "issue": [ + { + "number": 123, + "labels": { + "version": "2.2", + "backport": [ + "2.0", + "1.9" + ], + "category": { + "type": "API", + "level": "bug" + }, + "createdDate": "2023-01-01", + "comment": [ [ "Doe","Shipped" ],[ "John","Approved" ] ], + "views": 288, + "priority": 5.00 + } + } + ] + } + + - do: + index: + index: test_partial_flat_object_nested + id: 2 + body: { + "issue": [ + { + "number": 456, + "labels": { + "author": "Liu", + "version": "2.1", + "backport": [ + "2.0", + "1.3" + ], + "category": { + "type": "API", + "level": "enhancement" + }, + "createdDate": "2023-02-01", + "comment": [ [ "Mike","LGTM" ],[ "John","Approved" ] ], + "views": 3333, + "priority": 1.50 + } + } + ] + } + + - do: + indices.refresh: + index: test_partial_flat_object_nested +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: test_partial_flat_object_nested + + +--- +# Verify that mappings under the catalog field did not expand +# and no dynamic fields were created. +"Mappings": + - skip: + version: " - 2.99.99" + reason: "flat_object is introduced in 3.0.0 in main branch" + + - do: + indices.get_mapping: + index: test_partial_flat_object_nested + - is_true: test_partial_flat_object_nested.mappings + - match: { test_partial_flat_object_nested.mappings.properties.issue.properties.number.type: integer } + - match: { test_partial_flat_object_nested.mappings.properties.issue.properties.labels.type: flat_object } + # https://github.com/opensearch-project/OpenSearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test#length + - length: { test_partial_flat_object_nested.mappings.properties.issue.properties: 2 } + - length: { test_partial_flat_object_nested.mappings.properties.issue.properties.labels: 1 } + + +--- +"Supported queries": + - skip: + version: " - 2.99.99" + reason: "flat_object is introduced in 3.0.0 in main branch" + + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 2 } + + # Match Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + match: { "issue.labels.version": "2.1" } + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.version: "2.1" } + + # Match Query without exact dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + match: { "issue.labels": "2.1" } + } + } + } + } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.version: "2.1" } + + # Term Query1 with exact dot path for date + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels.createdDate: "2023-01-01" } + } + } } } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.createdDate: "2023-01-01" } + + # Term Query1 without exact dot path for date + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels: "2023-01-01" } + } } } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.createdDate: "2023-01-01" } + + # Term Query2 with dot path for string + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { "issue.labels.category.type": "API" } + } } } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.category.type: "API" } + - match: { hits.hits.1._source.issue.0.labels.category.type: "API" } + + # Term Query2 without exact dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels: "API" } + } } } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.category.type: "API" } + - match: { hits.hits.1._source.issue.0.labels.category.type: "API" } + + # Term Query3 with dot path for array + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels.backport: "1.9" } + } } } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.backport: [ "2.0", "1.9" ] } + + # Term Query3 without dot path for array + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels: "1.9" } + } } } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.backport: [ "2.0", "1.9" ] } + + # Term Query4 with dot path for nested-array + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels.comment: "LGTM" } + } } } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Term Query4 without dot path for nested-array + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + term: { issue.labels: "LGTM" } } } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Terms Query without dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + terms: { issue.labels: [ "John","Mike" ] } } } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Terms Query with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + terms: { issue.labels.comment: [ "John","Mike" ] } } } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Prefix Query with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "prefix": { + "issue.labels.comment": { + "value": "Mi" + } } } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Prefix Query without dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "prefix": { + "issue.labels": { + "value": "Mi" + } + } + } } } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Range Query with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "range": { + "issue.labels.version": { + "gte": "2.1", + "lte": "3.0" + } } } + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.version: "2.2" } + - match: { hits.hits.1._source.issue.0.labels.version: "2.1" } + + # Range Query without dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "range": { + "issue.labels": { + "gte": "2.1", + "lte": "3.0" + } } } + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.version: "2.2" } + - match: { hits.hits.1._source.issue.0.labels.version: "2.1" } + + # Range Query with integer input with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "range": { + "issue.labels.views": { + "gte": 3000, + "lte": 4000 + } } } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.views: 3333 } + + # Range Query with integer input without dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "range": { + "issue.labels": { + "gte": 3000, + "lte": 4000 + } } } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.views: 3333 } + + + # Range Query with double input with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "range": { + "issue.labels.priority": { + "gte": 4.1234, + "lte": 5.1234 + } } } + } + } + } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.priority: 5.00 } + + # Range Query with double input without dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "range": { + "issue.labels": { + "gte": 4.1234, + "lte": 5.1234 + } } } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.priority: 5.00 } + + + # Exists Query with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "exists": { + "field": issue.labels.priority + } } } + } + } + + - length: { hits.hits: 2 } + + # Exists Query with nested dot path, use the flat_object_field_name.last_key + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "exists": { + "field": issue.labels.type + } } } + } + } + + - length: { hits.hits: 2 } + + # Exists Query without dot path for the flat_object_field_name + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "exists": { + "field": issue.labels + } } } + } + } + + - length: { hits.hits: 2 } + + # Exists Query2 with dot path for one hit + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "exists": { + "field": issue.labels.author + } } } + } + } + + - length: { hits.hits: 1 } + + # Query_string Query without dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "query_string": { + "fields": [ "issue.labels" ], + "query": "Doe OR Mike" + } } } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Query_string Query with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "query_string": { + "fields": [ "issue.labels.comment" ], + "query": "Doe OR Mike" + } } } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + - match: { hits.hits.1._source.issue.0.labels.comment: [ [ "Mike","LGTM" ],[ "John","Approved" ] ] } + + # Simple_query_string Query without full dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "simple_query_string": { + "query": "Doe", + "fields": [ "issue.labels" ] + } } } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + + # Simple_query_string Query with dot path. + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "simple_query_string": { + "query": "Doe", + "fields": [ "issue.labels.comment" ] + } } } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index e0b37df5c1734..36e0adbbf057f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -85,7 +85,7 @@ public static class Defaults { @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key); + return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name()); } /** @@ -186,6 +186,8 @@ public static final class FlatObjectFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; + private final String mappedFieldTypeName; + private KeywordFieldMapper.KeywordFieldType valueFieldType; private KeywordFieldMapper.KeywordFieldType valueAndPathFieldType; @@ -195,10 +197,7 @@ public FlatObjectFieldType(String name, boolean isSearchable, boolean hasDocValu setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; - } - - public FlatObjectFieldType(String name) { - this(name, true, true, Collections.emptyMap()); + this.mappedFieldTypeName = null; } public FlatObjectFieldType(String name, FieldType fieldType) { @@ -212,12 +211,28 @@ public FlatObjectFieldType(String name, FieldType fieldType) { ); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; + this.mappedFieldTypeName = null; } public FlatObjectFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; + this.mappedFieldTypeName = null; + } + + public FlatObjectFieldType(String name, String mappedFieldTypeName) { + super( + name, + true, + false, + true, + new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), + Collections.emptyMap() + ); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; + this.mappedFieldTypeName = mappedFieldTypeName; } void setValueFieldType(KeywordFieldMapper.KeywordFieldType valueFieldType) { @@ -356,22 +371,21 @@ public Query termsQuery(List values, QueryShardContext context) { * @return directedSubFieldName */ public String directSubfield() { - if (name().contains(DOT_SYMBOL)) { - String[] dotPathList = name().split("\\."); - return dotPathList[0] + VALUE_AND_PATH_SUFFIX; + if (mappedFieldTypeName == null) { + return new StringBuilder().append(this.name()).append(VALUE_SUFFIX).toString(); } else { - return this.valueFieldType.name(); + return new StringBuilder().append(this.mappedFieldTypeName).append(VALUE_AND_PATH_SUFFIX).toString(); } } /** - * If the search key is assigned with value, - * the dot path was used in search query, then - * rewrite the searchValueString as the format "dotpath=value", + * If the search key has mappedFieldTypeName as prefix, + * then the dot path was used in search query, + * then rewrite the searchValueString as the format "dotpath=value", * @return rewriteSearchValue */ public String rewriteValue(String searchValueString) { - if (!name().contains(DOT_SYMBOL)) { + if (!hasMappedFieldTyeNameInQueryFieldName(name())) { return searchValueString; } else { String rewriteSearchValue = new StringBuilder().append(name()).append(EQUAL_SYMBOL).append(searchValueString).toString(); @@ -380,6 +394,23 @@ public String rewriteValue(String searchValueString) { } + private boolean hasMappedFieldTyeNameInQueryFieldName(String input) { + String prefix = this.mappedFieldTypeName; + if (prefix == null) { + return false; + } + if (!input.startsWith(prefix)) { + return false; + } + String rest = input.substring(prefix.length()); + + if (rest.isEmpty()) { + return false; + } else { + return true; + } + } + private String inputToString(Object inputValue) { if (inputValue instanceof Integer) { String inputToString = Integer.toString((Integer) inputValue); @@ -460,15 +491,15 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } /** - * if there is dot path. query the field name in flatObject parent field. + * if there is dot path. query the field name in flatObject parent field (mappedFieldTypeName). * else query in _field_names system field */ @Override public Query existsQuery(QueryShardContext context) { String searchKey; String searchField; - if (name().contains(DOT_SYMBOL)) { - searchKey = name().split("\\.")[0]; + if (hasMappedFieldTyeNameInQueryFieldName(name())) { + searchKey = this.mappedFieldTypeName; searchField = name(); } else { searchKey = FieldNamesFieldMapper.NAME; diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java new file mode 100644 index 0000000000000..9ec053dc59d10 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java @@ -0,0 +1,134 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.analysis.AnalyzerScope; +import org.opensearch.index.analysis.NamedAnalyzer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class FlatObjectFieldTypeTests extends FieldTypeTestCase { + private static MappedFieldType getFlatParentFieldType(String fieldName) { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + MappedFieldType flatParentFieldType = new FlatObjectFieldMapper.Builder(fieldName).build(context).fieldType(); + return flatParentFieldType; + } + + public void testFetchSourceValue() throws IOException { + MappedFieldType mapper = getFlatParentFieldType("field"); + + Map jsonPoint = new HashMap<>(); + jsonPoint.put("type", "flat_object"); + jsonPoint.put("coordinates", Arrays.asList(42.0, 27.1)); + Map otherJsonPoint = new HashMap<>(); + otherJsonPoint.put("type", "Point"); + otherJsonPoint.put("coordinates", Arrays.asList(30.0, 50.0)); + + ArrayList jsonPointList = new ArrayList<>(); + jsonPointList.add(jsonPoint.toString()); + + ArrayList otherJsonPointList = new ArrayList<>(); + otherJsonPointList.add(otherJsonPoint.toString()); + + assertEquals(jsonPointList, fetchSourceValue(mapper, jsonPoint, null)); + assertEquals(otherJsonPointList, fetchSourceValue(mapper, otherJsonPoint, null)); + + } + + public void testDirectSubfield() { + { + MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + + // when searching for "foo" in "field", the directSubfield is field._value field + String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); + assertEquals("field._value", searchFieldName); + + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("bar", flatParentFieldType.name()); + // when searching for "foo" in "field.bar", the directSubfield is field._valueAndPath field + String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); + assertEquals("field._valueAndPath", searchFieldNameDocPath); + } + { + NamedAnalyzer analyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, null); + MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", analyzer); + assertEquals("field._value", ((FlatObjectFieldMapper.FlatObjectFieldType) ft).directSubfield()); + } + } + + public void testRewriteValue() { + MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + + // when searching for "foo" in "field", the rewrite value is "foo" + String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); + assertEquals("foo", searchValues); + + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); + + // when searching for "foo" in "field.bar", the rewrite value is "field.bar=foo" + String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); + String searchValuesDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).rewriteValue("foo"); + assertEquals("field.bar=foo", searchValuesDocPath); + } + + public void testTermQuery() { + + MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + + // when searching for "foo" in "field", the term query is directed to search "foo" in field._value field + String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); + String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); + assertEquals("foo", searchValues); + assertEquals(new TermQuery(new Term(searchFieldName, searchValues)), flatParentFieldType.termQuery(searchValues, null)); + + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); + + // when searching for "foo" in "field.bar", the term query is directed to search in field._valueAndPath field + String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); + String searchValuesDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).rewriteValue("foo"); + assertEquals("field.bar=foo", searchValuesDocPath); + assertEquals(new TermQuery(new Term(searchFieldNameDocPath, searchValuesDocPath)), dynamicMappedFieldType.termQuery("foo", null)); + + MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType("field", false, true, Collections.emptyMap()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("bar", null)); + assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + } + + public void testExistsQuery() { + { + MappedFieldType ft = getFlatParentFieldType("field"); + // when checking on the flat_object field name "field", check if exist in the field mapper names + assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); + + // when checking if a subfield within the flat_object, for example, "field.bar", use term query in the flat_object field + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", ft.name()); + assertEquals(new TermQuery(new Term("field", "field.bar")), dynamicMappedFieldType.existsQuery(null)); + + } + { + FlatObjectFieldMapper.FlatObjectFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType( + "field", + true, + false, + Collections.emptyMap() + ); + assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); + } + } +} From 0e037a2e77a146ac13c608d0dcc22597c7020863 Mon Sep 17 00:00:00 2001 From: Bansi Kasundra <66969140+kasundra07@users.noreply.github.com> Date: Thu, 6 Jul 2023 21:11:31 -0700 Subject: [PATCH 064/186] Add Changes in Snapshot Clone Flow for remote store interoperability. (#7496) Signed-off-by: Bansi Kasundra --- .../opensearch/snapshots/CloneSnapshotIT.java | 161 +++++++++++++++++ .../cluster/SnapshotsInProgress.java | 23 ++- .../RemoteStoreShardShallowCopySnapshot.java | 25 +++ .../index/store/lockmanager/FileLockInfo.java | 13 +- .../lockmanager/RemoteStoreLockManager.java | 10 ++ .../RemoteStoreMetadataLockManager.java | 40 +++-- .../repositories/FilterRepository.java | 13 ++ .../opensearch/repositories/Repository.java | 22 +++ .../blobstore/BlobStoreRepository.java | 52 +++++- .../opensearch/snapshots/SnapshotInfo.java | 1 + .../snapshots/SnapshotsService.java | 166 ++++++++++++------ .../store/lockmanager/FileLockInfoTests.java | 9 +- .../RepositoriesServiceTests.java | 13 ++ .../index/shard/RestoreOnlyRepository.java | 13 ++ .../AbstractSnapshotIntegTestCase.java | 40 +++++ 15 files changed, 530 insertions(+), 71 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 5441dae9703ce..ce92a15026b70 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -38,11 +38,15 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.SnapshotsInProgress; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexNotFoundException; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryData; import org.opensearch.snapshots.mockstore.MockRepository; +import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; import java.util.ArrayList; @@ -64,6 +68,7 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.nio.file.Path; +import java.util.concurrent.ExecutionException; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -153,6 +158,162 @@ public void testCloneSnapshotIndex() throws Exception { assertEquals(status1.getStats().getTotalSize(), status2.getStats().getTotalSize()); } + public void testCloneShallowSnapshotIndex() throws Exception { + disableRepoConsistencyCheck("This test uses remote store repository"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "fs", snapshotRepoPath); + + final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; + final Path shallowSnapshotRepoPath = randomRepoPath(); + createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String snapshot = "snapshot"; + createFullSnapshot(snapshotRepoName, snapshot); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + + indexRandomDocs(indexName, randomIntBetween(20, 100)); + + final String shallowSnapshot = "shallow-snapshot"; + createFullSnapshot(shallowSnapshotRepoName, shallowSnapshot); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + + if (randomBoolean()) { + assertAcked(admin().indices().prepareDelete(indexName)); + } + + final String sourceSnapshot = shallowSnapshot; + final String targetSnapshot = "target-snapshot"; + assertAcked(startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get()); + logger.info("Lock files count: {}", getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); + } + + public void testShallowCloneNameAvailability() throws Exception { + disableRepoConsistencyCheck("This test uses remote store repository"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); + internalCluster().startDataOnlyNode(); + + final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; + final Path shallowSnapshotRepoPath = randomRepoPath(); + createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String shallowSnapshot1 = "snapshot1"; + createFullSnapshot(shallowSnapshotRepoName, shallowSnapshot1); + + final String shallowSnapshot2 = "snapshot2"; + createFullSnapshot(shallowSnapshotRepoName, shallowSnapshot2); + + ExecutionException ex = expectThrows( + ExecutionException.class, + () -> startClone(shallowSnapshotRepoName, shallowSnapshot1, shallowSnapshot2, indexName, remoteStoreEnabledIndexName).get() + ); + assertThat(ex.getMessage(), containsString("snapshot with the same name already exists")); + } + + public void testCloneAfterRepoShallowSettingEnabled() throws Exception { + disableRepoConsistencyCheck("This test uses remote store repository"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "fs", snapshotRepoPath); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String snapshot = "snapshot"; + createFullSnapshot(snapshotRepoName, snapshot); + assertEquals(getSnapshot(snapshotRepoName, snapshot).state(), SnapshotState.SUCCESS); + + // Updating the snapshot repository flag to enable shallow snapshots + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); + RepositoryMetadata updatedRepositoryMetadata = clusterAdmin().prepareGetRepositories(snapshotRepoName).get().repositories().get(0); + assertTrue(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); + + final String targetSnapshot = "target-snapshot"; + assertAcked(startClone(snapshotRepoName, snapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get()); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + assertEquals(getSnapshot(snapshotRepoName, targetSnapshot).isRemoteStoreIndexShallowCopyEnabled(), false); + } + + public void testCloneAfterRepoShallowSettingDisabled() throws Exception { + disableRepoConsistencyCheck("This test uses remote store repository"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String snapshot = "snapshot"; + createFullSnapshot(snapshotRepoName, snapshot); + assertEquals(getSnapshot(snapshotRepoName, snapshot).state(), SnapshotState.SUCCESS); + + // Updating the snapshot repository flag to enable shallow snapshots + createRepository(snapshotRepoName, "fs", snapshotRepoPath); + RepositoryMetadata updatedRepositoryMetadata = clusterAdmin().prepareGetRepositories(snapshotRepoName).get().repositories().get(0); + assertFalse(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); + + final String targetSnapshot = "target-snapshot"; + assertAcked(startClone(snapshotRepoName, snapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get()); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); + assertEquals(getSnapshot(snapshotRepoName, targetSnapshot).isRemoteStoreIndexShallowCopyEnabled(), true); + } + public void testClonePreventsSnapshotDelete() throws Exception { final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode(); diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index d827dc6409778..ffc99c34fcac5 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -167,7 +167,8 @@ public static Entry startClone( version, source, Map.of(), - false // TODO: need to pull this value from the original snapshot, use whatever we set during snapshot create. + false // initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with + // clone jobs ); } @@ -453,6 +454,26 @@ public Entry withClones(final Map update ); } + public Entry withRemoteStoreIndexShallowCopy(final boolean remoteStoreIndexShallowCopy) { + return new Entry( + snapshot, + includeGlobalState, + partial, + state, + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + failure, + userMetadata, + version, + source, + clones, + remoteStoreIndexShallowCopy + ); + } + /** * Create a new instance by aborting this instance. Moving all in-progress shards to {@link ShardState#ABORTED} if assigned to a * data node or to {@link ShardState#FAILED} if not assigned to any data node. diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java index 8e6ed870c904f..8cb9fd3cd3c63 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java @@ -408,4 +408,29 @@ private void verifyParameters( throw new IllegalArgumentException(exceptionStr); } } + + /** + * Creates a new instance which has a different name and zero incremental file counts but is identical to this instance in terms of the files + * it references. + * + * @param targetSnapshotName target snapshot name + * @param startTime time the clone operation on the repository was started + * @param time time it took to create the clone + */ + public RemoteStoreShardShallowCopySnapshot asClone(String targetSnapshotName, long startTime, long time) { + return new RemoteStoreShardShallowCopySnapshot( + targetSnapshotName, + indexVersion, + primaryTerm, + commitGeneration, + startTime, + time, + totalFileCount, + totalSize, + indexUUID, + remoteStoreRepository, + repositoryBasePath, + fileNames + ); + } } diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java b/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java index a8fb7bf20c393..24f42743e1a04 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java @@ -8,6 +8,7 @@ package org.opensearch.index.store.lockmanager; +import java.nio.file.NoSuchFileException; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -50,13 +51,21 @@ String getLockPrefix() { return fileToLock + RemoteStoreLockManagerUtils.SEPARATOR; } - List getLocksForAcquirer(String[] lockFiles) { + String getLockForAcquirer(String[] lockFiles) throws NoSuchFileException { if (acquirerId == null || acquirerId.isBlank()) { throw new IllegalArgumentException("Acquirer ID should be provided"); } - return Arrays.stream(lockFiles) + List locksForAcquirer = Arrays.stream(lockFiles) .filter(lockFile -> acquirerId.equals(LockFileUtils.getAcquirerIdFromLock(lockFile))) .collect(Collectors.toList()); + + if (locksForAcquirer.isEmpty()) { + throw new NoSuchFileException("No lock file found for the acquirer: " + acquirerId); + } + if (locksForAcquirer.size() != 1) { + throw new IllegalStateException("Expected single lock file but found [" + locksForAcquirer.size() + "] lock files"); + } + return locksForAcquirer.get(0); } public static LockInfoBuilder getLockInfoBuilder() { diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManager.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManager.java index c30be082b4795..9eb066d9e955e 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManager.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManager.java @@ -39,6 +39,16 @@ public interface RemoteStoreLockManager { */ Boolean isAcquired(LockInfo lockInfo) throws IOException; + /** + * Acquires lock on the file mentioned in originalLockInfo for acquirer mentioned in clonedLockInfo. + * There can occur a race condition where the original file is deleted before we can use it to acquire lock for the new acquirer. Until we have a + * fix on LockManager side, Implementors must ensure thread safety for this operation. + * @param originalLockInfo lock info instance for original lock. + * @param clonedLockInfo lock info instance for which lock needs to be cloned. + * @throws IOException throws IOException if originalResource itself do not have any lock. + */ + void cloneLock(LockInfo originalLockInfo, LockInfo clonedLockInfo) throws IOException; + /* Deletes all lock related files and directories */ diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java index 7df20cae10664..fd7906729e314 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java @@ -15,8 +15,9 @@ import org.opensearch.index.store.RemoteBufferedOutputDirectory; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.util.Collection; -import java.util.List; +import java.util.Objects; /** * A Class that implements Remote Store Lock Manager by creating lock files for the remote store files that needs to @@ -48,6 +49,7 @@ public void acquire(LockInfo lockInfo) throws IOException { /** * Releases Locks acquired by a given acquirer which is passed in LockInfo Instance. + * If the lock file doesn't exist for the acquirer, release will be a no-op. * Right now this method is only used to release locks for a given acquirer, * This can be extended in future to handle other cases as well, like: * - release lock for given fileToLock and AcquirerId @@ -59,15 +61,12 @@ public void acquire(LockInfo lockInfo) throws IOException { public void release(LockInfo lockInfo) throws IOException { assert lockInfo instanceof FileLockInfo : "lockInfo should be instance of FileLockInfo"; String[] lockFiles = lockDirectory.listAll(); - - // ideally there should be only one lock per acquirer, but just to handle any stale locks, - // we try to release all the locks for the acquirer. - List locksToRelease = ((FileLockInfo) lockInfo).getLocksForAcquirer(lockFiles); - if (locksToRelease.size() > 1) { - logger.warn(locksToRelease.size() + " locks found for acquirer " + ((FileLockInfo) lockInfo).getAcquirerId()); - } - for (String lock : locksToRelease) { - lockDirectory.deleteFile(lock); + try { + String lockToRelease = ((FileLockInfo) lockInfo).getLockForAcquirer(lockFiles); + lockDirectory.deleteFile(lockToRelease); + } catch (NoSuchFileException e) { + // Ignoring if the file to be deleted is not present. + logger.info("No lock file found for acquirerId: {}", ((FileLockInfo) lockInfo).getAcquirerId()); } } @@ -84,6 +83,27 @@ public Boolean isAcquired(LockInfo lockInfo) throws IOException { return !lockFiles.isEmpty(); } + /** + * Acquires lock on the file mentioned in originalLockInfo for acquirer mentioned in clonedLockInfo. + * Snapshot layer enforces thread safety by having checks in place to ensure that the source snapshot is not being deleted before proceeding + * with the clone operation. Hence, the original lock file would always be present while acquiring the lock for cloned snapshot. + * @param originalLockInfo lock info instance for original lock. + * @param clonedLockInfo lock info instance for which lock needs to be cloned. + * @throws IOException throws IOException if originalResource itself do not have any lock. + */ + @Override + public void cloneLock(LockInfo originalLockInfo, LockInfo clonedLockInfo) throws IOException { + assert originalLockInfo instanceof FileLockInfo : "originalLockInfo should be instance of FileLockInfo"; + assert clonedLockInfo instanceof FileLockInfo : "clonedLockInfo should be instance of FileLockInfo"; + String originalResourceId = Objects.requireNonNull(((FileLockInfo) originalLockInfo).getAcquirerId()); + String clonedResourceId = Objects.requireNonNull(((FileLockInfo) clonedLockInfo).getAcquirerId()); + assert originalResourceId != null && clonedResourceId != null : "provided resourceIds should not be null"; + String[] lockFiles = lockDirectory.listAll(); + String lockNameForAcquirer = ((FileLockInfo) originalLockInfo).getLockForAcquirer(lockFiles); + String fileToLockName = FileLockInfo.LockFileUtils.getFileToLockNameFromLock(lockNameForAcquirer); + acquire(FileLockInfo.getLockInfoBuilder().withFileToLock(fileToLockName).withAcquirerId(clonedResourceId).build()); + } + public void delete() throws IOException { lockDirectory.delete(); } diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index b108e2da1ab04..764f36df6d337 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -47,6 +47,7 @@ import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; @@ -248,6 +249,18 @@ public void executeConsistentStateUpdate( in.executeConsistentStateUpdate(createUpdateTask, source, onFailure); } + @Override + public void cloneRemoteStoreIndexShardSnapshot( + SnapshotId source, + SnapshotId target, + RepositoryShardId shardId, + String shardGeneration, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + in.cloneRemoteStoreIndexShardSnapshot(source, target, shardId, shardGeneration, remoteStoreLockManagerFactory, listener); + } + @Override public void cloneShardSnapshot( SnapshotId source, diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index c08369b79452d..793b3d317e1bd 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -48,6 +48,7 @@ import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; @@ -375,6 +376,27 @@ void cloneShardSnapshot( ActionListener listener ); + /** + * Clones a remote store index shard snapshot. + * + * @param source source snapshot + * @param target target snapshot + * @param shardId shard id + * @param shardGeneration shard generation in repo + * @param remoteStoreLockManagerFactory remoteStoreLockManagerFactory for cloning metadata lock file + * @param listener listener to complete with new shard generation once clone has completed + */ + default void cloneRemoteStoreIndexShardSnapshot( + SnapshotId source, + SnapshotId target, + RepositoryShardId shardId, + @Nullable String shardGeneration, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + throw new UnsupportedOperationException(); + } + /** * Hook that allows a repository to filter the user supplied snapshot metadata in {@link SnapshotsInProgress.Entry#userMetadata()} * during snapshot initialization. diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index f04bf83c2f1d1..be5fbf2ab6a51 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -114,6 +114,9 @@ import org.opensearch.index.snapshots.blobstore.SnapshotFiles; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.index.store.lockmanager.FileLockInfo; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; +import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.repositories.IndexId; @@ -506,8 +509,8 @@ public void cloneShardSnapshot( executor.execute(ActionRunnable.supply(listener, () -> { final long startTime = threadPool.absoluteTimeInMillis(); final BlobContainer shardContainer = shardContainer(index, shardNum); - final BlobStoreIndexShardSnapshots existingSnapshots; final String newGen; + final BlobStoreIndexShardSnapshots existingSnapshots; final String existingShardGen; if (shardGeneration == null) { Tuple tuple = buildBlobStoreIndexShardSnapshots( @@ -560,6 +563,9 @@ public void cloneShardSnapshot( + "]. A snapshot by that name already exists for this shard." ); } + // We don't need to check if there exists a shallow snapshot with the same name as we have the check before starting the clone + // operation ensuring that the snapshot name is available by checking the repository data. Also, the new clone snapshot would + // have a different UUID and hence a new unique snap-N file will be created. final BlobStoreIndexShardSnapshot sourceMeta = loadShardSnapshot(shardContainer, source); logger.trace("[{}] [{}] writing shard snapshot file for clone", shardId, target); INDEX_SHARD_SNAPSHOT_FORMAT.write( @@ -578,6 +584,50 @@ public void cloneShardSnapshot( })); } + @Override + public void cloneRemoteStoreIndexShardSnapshot( + SnapshotId source, + SnapshotId target, + RepositoryShardId shardId, + @Nullable String shardGeneration, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + if (isReadOnly()) { + listener.onFailure(new RepositoryException(metadata.name(), "cannot clone shard snapshot on a readonly repository")); + return; + } + final IndexId index = shardId.index(); + final int shardNum = shardId.shardId(); + final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + executor.execute(ActionRunnable.supply(listener, () -> { + final long startTime = threadPool.relativeTimeInMillis(); + final BlobContainer shardContainer = shardContainer(index, shardNum); + // We don't need to check if there exists a shallow/full copy snapshot with the same name as we have the check before starting + // the clone operation ensuring that the snapshot name is available by checking the repository data. Also, the new clone shallow + // snapshot would have a different UUID and hence a new unique shallow-snap-N file will be created. + RemoteStoreShardShallowCopySnapshot remStoreBasedShardMetadata = loadShallowCopyShardSnapshot(shardContainer, source); + String indexUUID = remStoreBasedShardMetadata.getIndexUUID(); + String remoteStoreRepository = remStoreBasedShardMetadata.getRemoteStoreRepository(); + RemoteStoreMetadataLockManager remoteStoreMetadataLockManger = remoteStoreLockManagerFactory.newLockManager( + remoteStoreRepository, + indexUUID, + String.valueOf(shardId.shardId()) + ); + remoteStoreMetadataLockManger.cloneLock( + FileLockInfo.getLockInfoBuilder().withAcquirerId(source.getUUID()).build(), + FileLockInfo.getLockInfoBuilder().withAcquirerId(target.getUUID()).build() + ); + REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.write( + remStoreBasedShardMetadata.asClone(target.getName(), startTime, threadPool.absoluteTimeInMillis() - startTime), + shardContainer, + target.getUUID(), + compressor + ); + return shardGeneration; + })); + } + // Inspects all cluster state elements that contain a hint about what the current repository generation is and updates // #latestKnownRepoGen if a newer than currently known generation is found @Override diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java index 1619f36738f7b..6bdbcfee29a9a 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java @@ -532,6 +532,7 @@ public Boolean includeGlobalState() { return includeGlobalState; } + @Nullable public Boolean isRemoteStoreIndexShallowCopyEnabled() { return remoteStoreIndexShallowCopy; } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index b523c1ba12b05..0a455c376f62d 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -88,8 +88,10 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -151,6 +153,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final RepositoriesService repositoriesService; + private final RemoteStoreLockManagerFactory remoteStoreLockManagerFactory; + private final ThreadPool threadPool; private final Map>>> snapshotCompletionListeners = @@ -206,6 +210,7 @@ public SnapshotsService( this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.repositoriesService = repositoriesService; + this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(() -> repositoriesService); this.threadPool = transportService.getThreadPool(); this.transportService = transportService; @@ -621,7 +626,11 @@ public ClusterState execute(ClusterState currentState) { } } } - updatedEntry = cloneEntry.withClones(clonesBuilder); + updatedEntry = cloneEntry.withClones(clonesBuilder) + .withRemoteStoreIndexShallowCopy( + Boolean.TRUE.equals(snapshotInfoListener.result().isRemoteStoreIndexShallowCopyEnabled()) + ); + ; updatedEntries.set(i, updatedEntry); changed = true; break; @@ -649,7 +658,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS continue; } final RepositoryShardId repoShardId = indexClone.getKey(); - runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, repository); + final boolean remoteStoreIndexShallowCopy = Boolean.TRUE.equals(updatedEntry.remoteStoreIndexShallowCopy()); + runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, repository, remoteStoreIndexShallowCopy); } } else { // Extremely unlikely corner case of cluster-manager failing over between between starting the clone and @@ -667,60 +677,112 @@ private void runReadyClone( SnapshotId sourceSnapshot, ShardSnapshotStatus shardStatusBefore, RepositoryShardId repoShardId, - Repository repository + Repository repository, + boolean remoteStoreIndexShallowCopy ) { - final SnapshotId targetSnapshot = target.getSnapshotId(); - final String localNodeId = clusterService.localNode().getId(); - if (currentlyCloning.add(repoShardId)) { - repository.cloneShardSnapshot( - sourceSnapshot, - targetSnapshot, - repoShardId, - shardStatusBefore.generation(), - ActionListener.wrap( - generation -> innerUpdateSnapshotState( - new ShardSnapshotUpdate(target, repoShardId, new ShardSnapshotStatus(localNodeId, ShardState.SUCCESS, generation)), - ActionListener.runBefore( - ActionListener.wrap( - v -> logger.trace( - "Marked [{}] as successfully cloned from [{}] to [{}]", + final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + executor.execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + logger.warn( + "Failed to get repository data while cloning shard [{}] from [{}] to [{}]", + repoShardId, + sourceSnapshot, + target.getSnapshotId() + ); + failCloneShardAndUpdateClusterState(target, sourceSnapshot, repoShardId); + } + + @Override + protected void doRun() { + final String localNodeId = clusterService.localNode().getId(); + repository.getRepositoryData(ActionListener.wrap(repositoryData -> { + try { + final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData( + repositoryData, + sourceSnapshot, + repoShardId.index() + ); + final boolean cloneRemoteStoreIndexShardSnapshot = remoteStoreIndexShallowCopy + && indexMetadata.getSettings().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); + final SnapshotId targetSnapshot = target.getSnapshotId(); + final ActionListener listener = ActionListener.wrap( + generation -> innerUpdateSnapshotState( + new ShardSnapshotUpdate( + target, repoShardId, - sourceSnapshot, - targetSnapshot + new ShardSnapshotStatus(localNodeId, ShardState.SUCCESS, generation) ), - e -> { - logger.warn("Cluster state update after successful shard clone [{}] failed", repoShardId); - failAllListenersOnMasterFailOver(e); - } + ActionListener.runBefore( + ActionListener.wrap( + v -> logger.trace( + "Marked [{}] as successfully cloned from [{}] to [{}]", + repoShardId, + sourceSnapshot, + targetSnapshot + ), + e -> { + logger.warn("Cluster state update after successful shard clone [{}] failed", repoShardId); + failAllListenersOnMasterFailOver(e); + } + ), + () -> currentlyCloning.remove(repoShardId) + ) ), - () -> currentlyCloning.remove(repoShardId) - ) - ), - e -> innerUpdateSnapshotState( - new ShardSnapshotUpdate( - target, - repoShardId, - new ShardSnapshotStatus(localNodeId, ShardState.FAILED, "failed to clone shard snapshot", null) - ), - ActionListener.runBefore( - ActionListener.wrap( - v -> logger.trace( - "Marked [{}] as failed clone from [{}] to [{}]", + e -> { + logger.warn("Exception [{}] while trying to clone shard [{}]", e, repoShardId); + failCloneShardAndUpdateClusterState(target, sourceSnapshot, repoShardId); + } + ); + if (currentlyCloning.add(repoShardId)) { + if (cloneRemoteStoreIndexShardSnapshot) { + repository.cloneRemoteStoreIndexShardSnapshot( + sourceSnapshot, + targetSnapshot, repoShardId, + shardStatusBefore.generation(), + remoteStoreLockManagerFactory, + listener + ); + } else { + repository.cloneShardSnapshot( sourceSnapshot, - targetSnapshot - ), - ex -> { - logger.warn("Cluster state update after failed shard clone [{}] failed", repoShardId); - failAllListenersOnMasterFailOver(ex); - } - ), - () -> currentlyCloning.remove(repoShardId) - ) - ) - ) - ); - } + targetSnapshot, + repoShardId, + shardStatusBefore.generation(), + listener + ); + } + } + } catch (IOException e) { + logger.warn("Failed to get index-metadata from repository data for index [{}]", repoShardId.index().getName()); + failCloneShardAndUpdateClusterState(target, sourceSnapshot, repoShardId); + } + }, this::onFailure)); + } + }); + } + + private void failCloneShardAndUpdateClusterState(Snapshot target, SnapshotId sourceSnapshot, RepositoryShardId repoShardId) { + // Stale blobs/lock-files will be cleaned up during delete/cleanup operation. + final String localNodeId = clusterService.localNode().getId(); + innerUpdateSnapshotState( + new ShardSnapshotUpdate( + target, + repoShardId, + new ShardSnapshotStatus(localNodeId, ShardState.FAILED, "failed to clone shard snapshot", null) + ), + ActionListener.runBefore( + ActionListener.wrap( + v -> logger.trace("Marked [{}] as failed clone from [{}] to [{}]", repoShardId, sourceSnapshot, target.getSnapshotId()), + ex -> { + logger.warn("Cluster state update after failed shard clone [{}] failed", repoShardId); + failAllListenersOnMasterFailOver(ex); + } + ), + () -> currentlyCloning.remove(repoShardId) + ) + ); } private void ensureBelowConcurrencyLimit( @@ -3077,12 +3139,14 @@ private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, @Nul // this is a clone, see if new work is ready for (final Map.Entry clone : entry.clones().entrySet()) { if (clone.getValue().state() == ShardState.INIT) { + final boolean remoteStoreIndexShallowCopy = Boolean.TRUE.equals(entry.remoteStoreIndexShallowCopy()); runReadyClone( entry.snapshot(), entry.source(), clone.getValue(), clone.getKey(), - repositoriesService.repository(entry.repository()) + repositoriesService.repository(entry.repository()), + remoteStoreIndexShallowCopy ); } } diff --git a/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java b/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java index 95af53cb6e5ec..f3a2f1859923e 100644 --- a/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java +++ b/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java @@ -10,7 +10,7 @@ import org.opensearch.test.OpenSearchTestCase; -import java.util.List; +import java.nio.file.NoSuchFileException; public class FileLockInfoTests extends OpenSearchTestCase { String testMetadata = "testMetadata"; @@ -41,16 +41,13 @@ public void testGetLockPrefixFailureCase() { assertThrows(IllegalArgumentException.class, fileLockInfo::getLockPrefix); } - public void testGetLocksForAcquirer() { + public void testGetLocksForAcquirer() throws NoSuchFileException { String[] locks = new String[] { FileLockInfo.LockFileUtils.generateLockName(testMetadata, testAcquirerId), FileLockInfo.LockFileUtils.generateLockName(testMetadata, "acquirerId2") }; FileLockInfo fileLockInfo = FileLockInfo.getLockInfoBuilder().withAcquirerId(testAcquirerId).build(); - assertEquals( - fileLockInfo.getLocksForAcquirer(locks), - List.of(FileLockInfo.LockFileUtils.generateLockName(testMetadata, testAcquirerId)) - ); + assertEquals(fileLockInfo.getLockForAcquirer(locks), FileLockInfo.LockFileUtils.generateLockName(testMetadata, testAcquirerId)); } } diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index f5295bead19a4..edf5b6c84bc54 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -60,6 +60,7 @@ import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; @@ -378,6 +379,18 @@ public void cloneShardSnapshot( } + @Override + public void cloneRemoteStoreIndexShardSnapshot( + SnapshotId source, + SnapshotId target, + RepositoryShardId shardId, + String shardGeneration, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + + } + @Override public Lifecycle.State lifecycleState() { return null; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java index 2a85fffa8699a..67f3f7fc1f50c 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java @@ -44,6 +44,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.store.Store; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.IndexMetaDataGenerations; import org.opensearch.repositories.Repository; @@ -205,4 +206,16 @@ public void cloneShardSnapshot( ) { throw new UnsupportedOperationException("Unsupported for restore-only repository"); } + + @Override + public void cloneRemoteStoreIndexShardSnapshot( + SnapshotId source, + SnapshotId target, + RepositoryShardId shardId, + String shardGeneration, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + throw new UnsupportedOperationException("Unsupported for restore-only repository"); + } } diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index ddf9f3e96b9b4..83051c7fed4e4 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -51,6 +51,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorType; import org.opensearch.common.settings.Settings; @@ -61,6 +63,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.index.IndexModule; +import org.opensearch.index.store.RemoteBufferedOutputDirectory; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.NodeClosedException; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; @@ -511,6 +516,26 @@ protected void indexRandomDocs(String index, int numdocs) throws InterruptedExce assertDocCount(index, numdocs); } + protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") + .put("index.refresh_interval", "300s") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo) + .build(); + } + + protected Settings.Builder snapshotRepoSettingsForShallowCopy(Path path) { + final Settings.Builder settings = Settings.builder(); + settings.put("location", path); + settings.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE); + return settings; + } + protected long getCountForIndex(String indexName) { return client().search( new SearchRequest(new SearchRequest(indexName).source(new SearchSourceBuilder().size(0).trackTotalHits(true))) @@ -521,6 +546,21 @@ protected void assertDocCount(String index, long count) { assertEquals(getCountForIndex(index), count); } + protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepositoryName) throws IOException { + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreIndex) + .get() + .getSetting(remoteStoreIndex, IndexMetadata.SETTING_INDEX_UUID); + final RepositoriesService repositoriesService = internalCluster().getCurrentClusterManagerNodeInstance(RepositoriesService.class); + final BlobStoreRepository remoteStoreRepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepositoryName); + BlobPath shardLevelBlobPath = remoteStoreRepository.basePath().add(indexUUID).add("0").add("segments").add("lock_files"); + BlobContainer blobContainer = remoteStoreRepository.blobStore().blobContainer(shardLevelBlobPath); + try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) { + return lockDirectory.listAll(); + } + } + /** * Adds a snapshot in state {@link SnapshotState#FAILED} to the given repository. * From fc6a2ae2d7f58b3a9536efb8eeaa7b156c1be7b1 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 7 Jul 2023 12:30:45 +0530 Subject: [PATCH 065/186] Mute testStaleCommitDeletion test cases (#8516) Signed-off-by: Sachin Kale --- .../java/org/opensearch/remotestore/RemoteStoreIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index be8976671d04f..85208a33cc9f5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -285,6 +285,7 @@ public void testRemoteTranslogCleanup() throws Exception { verifyRemoteStoreCleanup(true); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8504") public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); @@ -309,6 +310,7 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { }, 30, TimeUnit.SECONDS); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8504") public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); From ca7045e8471c7394e502744703628c8d8adec445 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 7 Jul 2023 17:42:42 +0530 Subject: [PATCH 066/186] Compression Levels Settings for zstd and zstd_no_dict (#8312) * Enabling compression levels as index settings Signed-off-by: Sarthak Aggarwal * addressing review comments Signed-off-by: Sarthak Aggarwal * refactoring codec service Signed-off-by: Sarthak Aggarwal * index settings assertion Signed-off-by: Sarthak Aggarwal * default compression level change in the settings Signed-off-by: Sarthak Aggarwal --------- Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 1 + .../index/codec/CorrelationCodecService.java | 2 +- .../indices/IndexingMemoryControllerIT.java | 2 +- .../common/settings/IndexScopedSettings.java | 1 + .../opensearch/index/codec/CodecService.java | 19 +++++-- .../Lucene95CustomStoredFieldsFormat.java | 6 ++ .../index/codec/customcodecs/ZstdCodec.java | 4 +- .../codec/customcodecs/ZstdNoDictCodec.java | 4 +- .../opensearch/index/engine/EngineConfig.java | 13 +++++ .../opensearch/index/shard/IndexShard.java | 2 +- .../gateway/PrimaryShardAllocatorTests.java | 11 +++- .../opensearch/index/codec/CodecTests.java | 57 ++++++++++++++++--- ...Lucene95CustomStoredFieldsFormatTests.java | 20 +++++++ .../engine/EngineConfigFactoryTests.java | 8 ++- .../index/engine/InternalEngineTests.java | 8 +-- .../index/shard/IndexShardTests.java | 2 +- .../index/shard/RefreshListenersTests.java | 2 +- .../IndexingMemoryControllerTests.java | 2 +- .../OngoingSegmentReplicationsTests.java | 2 +- .../replication/common/CopyStateTests.java | 11 +++- .../index/engine/EngineTestCase.java | 12 ++-- 21 files changed, 148 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffc9e6d2783ea..045a1cebf4283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -155,6 +155,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) - Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) - [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) +- Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) ### Deprecated diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java index 0b70e7ed66f3d..09d5e1d2c19e3 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java @@ -27,7 +27,7 @@ public class CorrelationCodecService extends CodecService { * @param codecServiceConfig Generic codec service config */ public CorrelationCodecService(CodecServiceConfig codecServiceConfig) { - super(codecServiceConfig.getMapperService(), codecServiceConfig.getLogger()); + super(codecServiceConfig.getMapperService(), codecServiceConfig.getIndexSettings(), codecServiceConfig.getLogger()); mapperService = codecServiceConfig.getMapperService(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java index 72de689562e7a..c842ee8564041 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndexingMemoryControllerIT.java @@ -89,7 +89,7 @@ EngineConfig engineConfigWithLargerIndexingMemory(EngineConfig config) { .mergePolicy(config.getMergePolicy()) .analyzer(config.getAnalyzer()) .similarity(config.getSimilarity()) - .codecService(new CodecService(null, LogManager.getLogger(IndexingMemoryControllerIT.class))) + .codecService(new CodecService(null, indexSettings, LogManager.getLogger(IndexingMemoryControllerIT.class))) .eventListener(config.getEventListener()) .queryCache(config.getQueryCache()) .queryCachingPolicy(config.getQueryCachingPolicy()) diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 08d8199afface..739982036c2af 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -188,6 +188,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING, Store.FORCE_RAM_TERM_DICT, EngineConfig.INDEX_CODEC_SETTING, + EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING, EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS, IndexSettings.DEFAULT_PIPELINE, diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index b6dac7bd1596c..da64e2954b0d8 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -38,12 +38,15 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.customcodecs.ZstdCodec; import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + /** * Since Lucene 4.0 low level index segments are read and written through a * codec layer that allows to use use-case specific file formats & @@ -58,23 +61,27 @@ public class CodecService { public static final String DEFAULT_CODEC = "default"; public static final String BEST_COMPRESSION_CODEC = "best_compression"; - /** the raw unfiltered lucene default. useful for testing */ + /** + * the raw unfiltered lucene default. useful for testing + */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; public static final String ZSTD_CODEC = "zstd"; public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; - public CodecService(@Nullable MapperService mapperService, Logger logger) { + public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); + assert null != indexSettings; + int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); if (mapperService == null) { codecs.put(DEFAULT_CODEC, new Lucene95Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZSTD_CODEC, new ZstdCodec()); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec()); + codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger)); + codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java index f70306afc8562..2816e2907a5f6 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java @@ -35,6 +35,7 @@ public class Lucene95CustomStoredFieldsFormat extends StoredFieldsFormat { private final CompressionMode zstdNoDictCompressionMode; private final Lucene95CustomCodec.Mode mode; + private final int compressionLevel; /** default constructor */ public Lucene95CustomStoredFieldsFormat() { @@ -58,6 +59,7 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { */ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; zstdCompressionMode = new ZstdCompressionMode(compressionLevel); zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); } @@ -122,4 +124,8 @@ StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { Lucene95CustomCodec.Mode getMode() { return mode; } + + public int getCompressionLevel() { + return compressionLevel; + } } diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java index 68da782421e6e..04c110fceacdf 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java @@ -32,8 +32,8 @@ public ZstdCodec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } - public ZstdCodec(MapperService mapperService, Logger logger) { - super(Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL, mapperService, logger); + public ZstdCodec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD, compressionLevel, mapperService, logger); } /** The name for this codec. */ diff --git a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java index 26620473ec116..134f9a14422ad 100644 --- a/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java @@ -32,8 +32,8 @@ public ZstdNoDictCodec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } - public ZstdNoDictCodec(MapperService mapperService, Logger logger) { - super(Mode.ZSTD_NO_DICT, DEFAULT_COMPRESSION_LEVEL, mapperService, logger); + public ZstdNoDictCodec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); } /** The name for this codec. */ diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 7419cf1dadea6..f4aa5e248ac31 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -143,6 +143,19 @@ public Supplier retentionLeasesSupplier() { } }, Property.IndexScope, Property.NodeScope); + /** + * Index setting to change the compression level of zstd and zstd_no_dict lucene codecs. + * Compression Level gives a trade-off between compression ratio and speed. The higher compression level results in higher compression ratio but slower compression and decompression speeds. + * This setting is not realtime updateable. + */ + public static final Setting INDEX_CODEC_COMPRESSION_LEVEL_SETTING = Setting.intSetting( + "index.codec.compression_level", + 3, + 1, + 6, + Property.IndexScope + ); + /** * Configures an index to optimize documents with auto generated ids for append only. If this setting is updated from false * to true might not take effect immediately. In other words, disabling the optimization will be immediately applied while diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 01c0a12d463ea..e7720e9343b80 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -368,7 +368,7 @@ public IndexShard( assert shardRouting.initializing(); this.shardRouting = shardRouting; final Settings settings = indexSettings.getSettings(); - this.codecService = new CodecService(mapperService, logger); + this.codecService = new CodecService(mapperService, indexSettings, logger); this.warmer = warmer; this.similarityService = similarityService; Objects.requireNonNull(store, "Store must be provided to the index shard"); diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java index d15bb49f5342a..9a264781b0644 100644 --- a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java @@ -60,7 +60,9 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; +import org.opensearch.env.Environment; import org.opensearch.env.ShardLockObtainFailedException; +import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; @@ -69,6 +71,7 @@ import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotShardSizeInfo; import org.junit.Before; +import org.opensearch.test.IndexSettingsModule; import java.util.Arrays; import java.util.Collections; @@ -803,21 +806,25 @@ public TestAllocator addData( } public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary) { + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", nodeSettings); return addData( node, allocationId, primary, - ReplicationCheckpoint.empty(shardId, new CodecService(null, null).codec("default").getName()), + ReplicationCheckpoint.empty(shardId, new CodecService(null, indexSettings, null).codec("default").getName()), null ); } public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary, @Nullable Exception storeException) { + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", nodeSettings); return addData( node, allocationId, primary, - ReplicationCheckpoint.empty(shardId, new CodecService(null, null).codec("default").getName()), + ReplicationCheckpoint.empty(shardId, new CodecService(null, indexSettings, null).codec("default").getName()), storeException ); } diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 016e785f8da13..b0d904392407c 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -83,11 +83,31 @@ public void testBestCompression() throws Exception { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdWithCompressionLevel() throws Exception { + int randomCompressionLevel = randomIntBetween(1, 6); + Codec codec = createCodecService(randomCompressionLevel).codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictWithCompressionLevel() throws Exception { + int randomCompressionLevel = randomIntBetween(1, 6); + Codec codec = createCodecService(randomCompressionLevel).codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testDefaultMapperServiceNull() throws Exception { @@ -103,17 +123,25 @@ public void testBestCompressionMapperServiceNull() throws Exception { public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testExceptionCodecNull() { assertThrows(IllegalArgumentException.class, () -> createCodecService(true).codec(null)); } + public void testExceptionIndexSettingsNull() { + assertThrows(AssertionError.class, () -> new CodecService(null, null, LogManager.getLogger("test"))); + } + // write some docs with it, inspect .si to see this was the used compression private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); @@ -130,17 +158,29 @@ private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expect } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { - + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); if (isMapperServiceNull) { - return new CodecService(null, LogManager.getLogger("test")); + return new CodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); } - Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); - IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); - SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); - IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; + return buildCodecService(nodeSettings); + } + + private CodecService createCodecService(int randomCompressionLevel) throws IOException { + Settings nodeSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .put("index.codec.compression_level", randomCompressionLevel) + .build(); + return buildCodecService(nodeSettings); + } + + private CodecService buildCodecService(Settings nodeSettings) throws IOException { + + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); + SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); + IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService( - settings, + indexSettings, indexAnalyzers, xContentRegistry(), similarityService, @@ -149,7 +189,7 @@ private CodecService createCodecService(boolean isMapperServiceNull) throws IOEx () -> false, null ); - return new CodecService(service, LogManager.getLogger("test")); + return new CodecService(service, indexSettings, LogManager.getLogger("test")); } private SegmentReader getSegmentReader(Codec codec) throws IOException { @@ -166,4 +206,5 @@ private SegmentReader getSegmentReader(Codec codec) throws IOException { dir.close(); return sr; } + } diff --git a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java index 4f23450ce0b39..e87fb56770e4c 100644 --- a/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -24,4 +24,24 @@ public void testZstdNoDictLucene95CustomCodecMode() { assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); } + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); + } + } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java index f8bedc76ea994..bf9a86cff8b76 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java @@ -178,7 +178,7 @@ public Optional getEngineFactory(final IndexSettings indexSetting @Override public Optional getCustomCodecService(IndexSettings indexSettings) { - return Optional.of(new CodecService(null, LogManager.getLogger(getClass()))); + return Optional.of(new CodecService(null, indexSettings, LogManager.getLogger(getClass()))); } @Override @@ -195,7 +195,7 @@ public Optional getEngineFactory(final IndexSettings indexSetting @Override public Optional getCustomCodecService(IndexSettings indexSettings) { - return Optional.of(new CodecService(null, LogManager.getLogger(getClass()))); + return Optional.of(new CodecService(null, indexSettings, LogManager.getLogger(getClass()))); } } @@ -207,7 +207,9 @@ public Optional getEngineFactory(final IndexSettings indexSetting @Override public Optional getCustomCodecServiceFactory(IndexSettings indexSettings) { - return Optional.of(config -> new CodecService(config.getMapperService(), LogManager.getLogger(getClass()))); + return Optional.of( + config -> new CodecService(config.getMapperService(), config.getIndexSettings(), LogManager.getLogger(getClass())) + ); } } diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 3bd8e09cab777..209c9925bb06d 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -3226,7 +3226,7 @@ public void testFailStart() throws IOException { } public void testSettings() { - CodecService codecService = new CodecService(null, logger); + CodecService codecService = new CodecService(null, engine.config().getIndexSettings(), logger); LiveIndexWriterConfig currentIndexWriterConfig = engine.getCurrentIndexWriterConfig(); assertEquals(engine.config().getCodec().getName(), codecService.codec(codecName).getName()); @@ -3696,7 +3696,7 @@ public void testRecoverFromForeignTranslog() throws IOException { .mergePolicy(newMergePolicy()) .analyzer(config.getAnalyzer()) .similarity(config.getSimilarity()) - .codecService(new CodecService(null, logger)) + .codecService(new CodecService(null, config.getIndexSettings(), logger)) .eventListener(config.getEventListener()) .queryCache(IndexSearcher.getDefaultQueryCache()) .queryCachingPolicy(IndexSearcher.getDefaultQueryCachingPolicy()) @@ -3738,7 +3738,7 @@ public CustomTranslogDeletionPolicy(IndexSettings indexSettings, Supplier Date: Fri, 7 Jul 2023 22:55:29 +0530 Subject: [PATCH 067/186] [Remote Store] Update Translog Metadata file name (#8350) --------- Signed-off-by: Gaurav Bafna --- .../index/translog/RemoteFsTranslog.java | 7 +- .../transfer/BlobStoreTransferService.java | 28 +++-- .../translog/transfer/TransferService.java | 13 +- .../transfer/TranslogTransferManager.java | 111 ++++++++++------- .../transfer/TranslogTransferMetadata.java | 38 +++--- .../index/translog/RemoteFSTranslogTests.java | 14 +-- .../TranslogTransferManagerTests.java | 113 ++++++++++++++---- .../index/shard/IndexShardTestCase.java | 23 +++- 8 files changed, 221 insertions(+), 126 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 04057b581e8d9..1e565b97387d1 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -385,7 +385,8 @@ public void trimUnreferencedReaders() throws IOException { } if (generationsToDelete.isEmpty() == false) { deleteRemoteGeneration(generationsToDelete); - deleteStaleRemotePrimaryTermsAndMetadataFiles(); + translogTransferManager.deleteStaleTranslogMetadataFilesAsync(remoteGenerationDeletionPermits::release); + deleteStaleRemotePrimaryTerms(); } else { remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); } @@ -409,7 +410,7 @@ private void deleteRemoteGeneration(Set generations) { *
* This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. */ - private void deleteStaleRemotePrimaryTermsAndMetadataFiles() { + private void deleteStaleRemotePrimaryTerms() { // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part // of older primary term. @@ -418,8 +419,6 @@ private void deleteStaleRemotePrimaryTermsAndMetadataFiles() { assert readers.isEmpty() == false : "Expected non-empty readers"; long minimumReferencedPrimaryTerm = readers.stream().map(BaseTranslogReader::getPrimaryTerm).min(Long::compare).get(); translogTransferManager.deletePrimaryTermsAsync(minimumReferencedPrimaryTerm); - // Second we delete all stale metadata files from remote store - translogTransferManager.deleteStaleTranslogMetadataFilesAsync(); } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index 88fe816ccb462..d9feb1a832681 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionRunnable; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; @@ -23,6 +24,8 @@ import java.util.List; import java.util.Set; +import static org.opensearch.common.blobstore.BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC; + /** * Service that handles remote transfer of translog and checkpoint files * @@ -114,17 +117,6 @@ public Set listAll(Iterable path) throws IOException { return blobStore.blobContainer((BlobPath) path).listBlobs().keySet(); } - @Override - public void listAllAsync(String threadpoolName, Iterable path, ActionListener> listener) { - threadPool.executor(threadpoolName).execute(() -> { - try { - listener.onResponse(listAll(path)); - } catch (IOException e) { - listener.onFailure(e); - } - }); - } - @Override public Set listFolders(Iterable path) throws IOException { return blobStore.blobContainer((BlobPath) path).children().keySet(); @@ -140,4 +132,18 @@ public void listFoldersAsync(String threadpoolName, Iterable path, Actio } }); } + + public void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener) { + blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder("", limit, LEXICOGRAPHIC, listener); + } + + public void listAllInSortedOrderAsync( + String threadpoolName, + Iterable path, + int limit, + ActionListener> listener + ) { + threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, limit, listener); }); + } + } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java index 6aca3055a3f53..0e6496042e3d8 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java @@ -9,6 +9,7 @@ package org.opensearch.index.translog.transfer; import org.opensearch.action.ActionListener; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import java.io.IOException; @@ -80,14 +81,6 @@ void uploadBlobAsync( */ Set listAll(Iterable path) throws IOException; - /** - * Lists the files and invokes the listener on success or failure - * @param threadpoolName threadpool type which will be used to list all files asynchronously. - * @param path the path to list - * @param listener the callback to be invoked once list operation completes successfully/fails. - */ - void listAllAsync(String threadpoolName, Iterable path, ActionListener> listener); - /** * Lists the folders inside the path. * @param path : the path @@ -114,4 +107,8 @@ void uploadBlobAsync( */ InputStream downloadBlob(Iterable path, String fileName) throws IOException; + void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener); + + void listAllInSortedOrderAsync(String threadpoolName, Iterable path, int limit, ActionListener> listener); + } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index f6405bc9b5c82..6da0ee5521738 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -15,6 +15,8 @@ import org.apache.lucene.store.OutputStreamIndexOutput; import org.opensearch.action.ActionListener; import org.opensearch.action.LatchedActionListener; +import org.opensearch.common.SetOnce; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.VersionedCodecStreamWrapper; @@ -42,8 +44,6 @@ import static org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import static org.opensearch.index.translog.transfer.FileSnapshot.TranslogFileSnapshot; -import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_FILENAME_COMPARATOR; -import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.getFileName; /** * The class responsible for orchestrating the transfer of a {@link TransferSnapshot} via a {@link TransferService} @@ -185,15 +185,39 @@ private void downloadToFS(String fileName, Path location, String primaryTerm) th } public TranslogTransferMetadata readMetadata() throws IOException { - return transferService.listAll(remoteMetadataTransferPath).stream().max(METADATA_FILENAME_COMPARATOR).map(filename -> { - try (InputStream inputStream = transferService.downloadBlob(remoteMetadataTransferPath, filename)) { - IndexInput indexInput = new ByteArrayIndexInput("metadata file", inputStream.readAllBytes()); - return metadataStreamWrapper.readStream(indexInput); - } catch (IOException e) { - logger.error(() -> new ParameterizedMessage("Exception while reading metadata file: {}", filename), e); - return null; - } - }).orElse(null); + SetOnce metadataSetOnce = new SetOnce<>(); + SetOnce exceptionSetOnce = new SetOnce<>(); + final CountDownLatch latch = new CountDownLatch(1); + LatchedActionListener> latchedActionListener = new LatchedActionListener<>( + ActionListener.wrap(blobMetadataList -> { + if (blobMetadataList.isEmpty()) return; + String filename = blobMetadataList.get(0).name(); + try (InputStream inputStream = transferService.downloadBlob(remoteMetadataTransferPath, filename)) { + IndexInput indexInput = new ByteArrayIndexInput("metadata file", inputStream.readAllBytes()); + metadataSetOnce.set(metadataStreamWrapper.readStream(indexInput)); + } catch (IOException e) { + logger.error(() -> new ParameterizedMessage("Exception while reading metadata file: {}", filename), e); + exceptionSetOnce.set(e); + } + }, e -> { + logger.error(() -> new ParameterizedMessage("Exception while listing metadata files "), e); + exceptionSetOnce.set((IOException) e); + }), + latch + ); + + try { + transferService.listAllInSortedOrder(remoteMetadataTransferPath, 1, latchedActionListener); + latch.await(); + } catch (InterruptedException e) { + throw new IOException("Exception while reading/downloading metadafile", e); + } + + if (exceptionSetOnce.get() != null) { + throw exceptionSetOnce.get(); + } + + return metadataSetOnce.get(); } private TransferFileSnapshot prepareMetadata(TransferSnapshot transferSnapshot) throws IOException { @@ -211,7 +235,7 @@ private TransferFileSnapshot prepareMetadata(TransferSnapshot transferSnapshot) translogTransferMetadata.setGenerationToPrimaryTermMapper(new HashMap<>(generationPrimaryTermMap)); return new TransferFileSnapshot( - getFileName(translogTransferMetadata.getPrimaryTerm(), translogTransferMetadata.getGeneration()), + translogTransferMetadata.getFileName(), getMetadataBytes(translogTransferMetadata), translogTransferMetadata.getPrimaryTerm() ); @@ -230,7 +254,7 @@ public byte[] getMetadataBytes(TranslogTransferMetadata metadata) throws IOExcep try ( OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput( "translog transfer metadata " + metadata.getPrimaryTerm(), - getFileName(metadata.getPrimaryTerm(), metadata.getGeneration()), + metadata.getFileName(), output, TranslogTransferMetadata.BUFFER_SIZE ) @@ -253,20 +277,14 @@ public byte[] getMetadataBytes(TranslogTransferMetadata metadata) throws IOExcep */ public void deleteGenerationAsync(long primaryTerm, Set generations, Runnable onCompletion) { List translogFiles = new ArrayList<>(); - List metadataFiles = new ArrayList<>(); generations.forEach(generation -> { // Add .ckp and .tlog file to translog file list which is located in basePath/ String ckpFileName = Translog.getCommitCheckpointFileName(generation); String translogFileName = Translog.getFilename(generation); translogFiles.addAll(List.of(ckpFileName, translogFileName)); - // Add metadata file tio metadata file list which is located in basePath/metadata - String metadataFileName = TranslogTransferMetadata.getFileName(primaryTerm, generation); - metadataFiles.add(metadataFileName); }); // Delete the translog and checkpoint files asynchronously deleteTranslogFilesAsync(primaryTerm, translogFiles, onCompletion); - // Delete the metadata files asynchronously - deleteMetadataFilesAsync(metadataFiles, onCompletion); } /** @@ -341,24 +359,37 @@ public void onFailure(Exception e) { }); } - public void deleteStaleTranslogMetadataFilesAsync() { - transferService.listAllAsync(ThreadPool.Names.REMOTE_PURGE, remoteMetadataTransferPath, new ActionListener<>() { - @Override - public void onResponse(Set metadataFiles) { - List sortedMetadataFiles = metadataFiles.stream().sorted(METADATA_FILENAME_COMPARATOR).collect(Collectors.toList()); - if (sortedMetadataFiles.size() <= 1) { - logger.trace("Remote Metadata file count is {}, so skipping deletion", sortedMetadataFiles.size()); - return; - } - List metadataFilesToDelete = sortedMetadataFiles.subList(0, sortedMetadataFiles.size() - 1); - deleteMetadataFilesAsync(metadataFilesToDelete); - } + public void deleteStaleTranslogMetadataFilesAsync(Runnable onCompletion) { + try { + transferService.listAllInSortedOrderAsync( + ThreadPool.Names.REMOTE_PURGE, + remoteMetadataTransferPath, + Integer.MAX_VALUE, + new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + List sortedMetadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + if (sortedMetadataFiles.size() <= 1) { + logger.trace("Remote Metadata file count is {}, so skipping deletion", sortedMetadataFiles.size()); + onCompletion.run(); + return; + } + List metadataFilesToDelete = sortedMetadataFiles.subList(1, sortedMetadataFiles.size()); + logger.trace("Deleting remote translog metadata files {}", metadataFilesToDelete); + deleteMetadataFilesAsync(metadataFilesToDelete, onCompletion); + } - @Override - public void onFailure(Exception e) { - logger.error("Exception occurred while listing translog metadata files from remote store", e); - } - }); + @Override + public void onFailure(Exception e) { + logger.error("Exception occurred while listing translog metadata files from remote store", e); + onCompletion.run(); + } + } + ); + } catch (Exception e) { + logger.error("Exception occurred while listing translog metadata files from remote store", e); + onCompletion.run(); + } } public void deleteTranslogFiles() throws IOException { @@ -407,14 +438,6 @@ public void onFailure(Exception e) { } } - /** - * Deletes metadata files asynchronously using the {@code REMOTE_PURGE} threadpool. - * @param metadataFilesToDelete list of metadata files to be deleted. - */ - private void deleteMetadataFilesAsync(List metadataFilesToDelete) { - deleteMetadataFilesAsync(metadataFilesToDelete, () -> {}); - } - /** * Deletes metadata files asynchronously using the {@code REMOTE_PURGE} threadpool. On success or failure, runs {@code onCompletion}. * diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index 7a2fee9a69d5e..75d6549b23f1e 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -9,9 +9,9 @@ package org.opensearch.index.translog.transfer; import org.opensearch.common.SetOnce; +import org.opensearch.index.remote.RemoteStoreUtils; import java.util.Arrays; -import java.util.Comparator; import java.util.Map; import java.util.Objects; @@ -42,13 +42,14 @@ public class TranslogTransferMetadata { static final String METADATA_CODEC = "md"; - public static final Comparator METADATA_FILENAME_COMPARATOR = new MetadataFilenameComparator(); + private final long createdAt; public TranslogTransferMetadata(long primaryTerm, long generation, long minTranslogGeneration, int count) { this.primaryTerm = primaryTerm; this.generation = generation; this.minTranslogGeneration = minTranslogGeneration; this.count = count; + this.createdAt = System.currentTimeMillis(); } public long getPrimaryTerm() { @@ -75,8 +76,19 @@ public Map getGenerationToPrimaryTermMapper() { return generationToPrimaryTermMapper.get(); } - public static String getFileName(long primaryTerm, long generation) { - return String.join(METADATA_SEPARATOR, Arrays.asList(String.valueOf(primaryTerm), String.valueOf(generation))); + /* + This should be used only at the time of creation. + */ + public String getFileName() { + return String.join( + METADATA_SEPARATOR, + Arrays.asList( + RemoteStoreUtils.invertLong(primaryTerm), + RemoteStoreUtils.invertLong(generation), + RemoteStoreUtils.invertLong(createdAt), + String.valueOf(CURRENT_VERSION) + ) + ); } @Override @@ -91,22 +103,4 @@ public boolean equals(Object o) { TranslogTransferMetadata other = (TranslogTransferMetadata) o; return Objects.equals(this.primaryTerm, other.primaryTerm) && Objects.equals(this.generation, other.generation); } - - private static class MetadataFilenameComparator implements Comparator { - @Override - public int compare(String first, String second) { - // Format of metadata filename is __ - String[] filenameTokens1 = first.split(METADATA_SEPARATOR); - String[] filenameTokens2 = second.split(METADATA_SEPARATOR); - // Here, we are comparing only primary term and generation. - for (int i = 0; i < filenameTokens1.length; i++) { - if (filenameTokens1[i].equals(filenameTokens2[i]) == false) { - return Long.compare(Long.parseLong(filenameTokens1[i]), Long.parseLong(filenameTokens2[i])); - } - } - throw new IllegalArgumentException( - "TranslogTransferMetadata files " + first + " and " + second + " have same primary term and generation" - ); - } - } } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index d963830e9e736..30c04c731d1f8 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -570,7 +570,7 @@ public void testMetadataFileDeletion() throws Exception { assertEquals(1, translog.readers.size()); } assertBusy(() -> assertEquals(4, translog.allUploaded().size())); - assertBusy(() -> assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); int moreDocs = randomIntBetween(3, 10); logger.info("numDocs={} moreDocs={}", numDocs, moreDocs); for (int i = numDocs; i < numDocs + moreDocs; i++) { @@ -579,7 +579,7 @@ public void testMetadataFileDeletion() throws Exception { translog.trimUnreferencedReaders(); assertEquals(1 + moreDocs, translog.readers.size()); assertBusy(() -> assertEquals(2 + 2L * moreDocs, translog.allUploaded().size())); - assertBusy(() -> assertEquals(1 + moreDocs, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); int totalDocs = numDocs + moreDocs; translog.setMinSeqNoToKeep(totalDocs - 1); @@ -592,7 +592,7 @@ public void testMetadataFileDeletion() throws Exception { ); translog.setMinSeqNoToKeep(totalDocs); translog.trimUnreferencedReaders(); - assertBusy(() -> assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); // Change primary term and test the deletion of older primaries String translogUUID = translog.translogUUID; @@ -607,10 +607,6 @@ public void testMetadataFileDeletion() throws Exception { long oldPrimaryTerm = primaryTerm.get(); long newPrimaryTerm = primaryTerm.incrementAndGet(); - // Check all metadata files corresponds to old primary term - Set mdFileNames = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); - assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(oldPrimaryTerm).concat("__")))); - // Creating RemoteFsTranslog with the same location Translog newTranslog = create(translogDir, repository, translogUUID); int newPrimaryTermDocs = randomIntBetween(5, 10); @@ -621,10 +617,6 @@ public void testMetadataFileDeletion() throws Exception { newTranslog.trimUnreferencedReaders(); } - // Check that all metadata files are belonging now to the new primary - mdFileNames = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); - assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(newPrimaryTerm).concat("__")))); - try { newTranslog.close(); } catch (Exception e) { diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 6f6b3622295b6..924a9d039da28 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -11,10 +11,12 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.mockito.Mockito; import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.util.set.Sets; +import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.transfer.FileSnapshot.CheckpointFileSnapshot; @@ -31,10 +33,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -186,11 +191,17 @@ public void testReadMetadataNoFile() throws IOException { remoteBaseTransferPath, null ); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(2); + List bmList = new LinkedList<>(); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); - when(transferService.listAll(remoteBaseTransferPath)).thenReturn(Sets.newHashSet()); assertNull(translogTransferManager.readMetadata()); } + // This should happen most of the time - Just a single metadata file public void testReadMetadataSingleFile() throws IOException { TranslogTransferManager translogTransferManager = new TranslogTransferManager( shardId, @@ -198,19 +209,25 @@ public void testReadMetadataSingleFile() throws IOException { remoteBaseTransferPath, null ); - - // BlobPath does not have equals method, so we can't use the instance directly in when - when(transferService.listAll(any(BlobPath.class))).thenReturn(Sets.newHashSet("12__234")); + TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2); + String mdFilename = tm.getFileName(); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(2); + List bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(mdFilename, 1)); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata(); - when(transferService.downloadBlob(any(BlobPath.class), eq("12__234"))).thenReturn( + when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenReturn( new ByteArrayInputStream(translogTransferManager.getMetadataBytes(metadata)) ); assertEquals(metadata, translogTransferManager.readMetadata()); } - public void testReadMetadataMultipleFiles() throws IOException { + public void testReadMetadataReadException() throws IOException { TranslogTransferManager translogTransferManager = new TranslogTransferManager( shardId, transferService, @@ -218,17 +235,33 @@ public void testReadMetadataMultipleFiles() throws IOException { null ); - when(transferService.listAll(any(BlobPath.class))).thenReturn(Sets.newHashSet("12__234", "12__235", "12__233")); + TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2); + String mdFilename = tm.getFileName(); - TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata(); - when(transferService.downloadBlob(any(BlobPath.class), eq("12__235"))).thenReturn( - new ByteArrayInputStream(translogTransferManager.getMetadataBytes(metadata)) - ); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(2); + List bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(mdFilename, 1)); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); - assertEquals(metadata, translogTransferManager.readMetadata()); + when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenThrow(new IOException("Something went wrong")); + + assertThrows(IOException.class, translogTransferManager::readMetadata); + } + + public void testMetadataFileNameOrder() throws IOException { + // asserting that new primary followed new generation are lexicographically smallest + String mdFilenameGen1 = new TranslogTransferMetadata(1, 1, 1, 2).getFileName(); + String mdFilenameGen2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName(); + String mdFilenamePrimary2 = new TranslogTransferMetadata(2, 1, 1, 2).getFileName(); + List metadataFiles = Arrays.asList(mdFilenameGen1, mdFilenameGen2, mdFilenamePrimary2); + Collections.sort(metadataFiles); + assertEquals(Arrays.asList(mdFilenamePrimary2, mdFilenameGen2, mdFilenameGen1), metadataFiles); } - public void testReadMetadataException() throws IOException { + public void testReadMetadataListException() throws IOException { TranslogTransferManager translogTransferManager = new TranslogTransferManager( shardId, transferService, @@ -236,16 +269,15 @@ public void testReadMetadataException() throws IOException { null ); - when(transferService.listAll(any(BlobPath.class))).thenReturn(Sets.newHashSet("12__234", "12__235", "12__233")); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(2); + latchedActionListener.onFailure(new IOException("Issue while listing")); + return null; + }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); - when(transferService.downloadBlob(any(BlobPath.class), eq("12__235"))).thenThrow(new IOException("Something went wrong")); + when(transferService.downloadBlob(any(BlobPath.class), any(String.class))).thenThrow(new IOException("Something went wrong")); - assertNull(translogTransferManager.readMetadata()); - } - - public void testReadMetadataSamePrimaryTermGeneration() throws IOException { - List metadataFiles = Arrays.asList("12__234", "12__235", "12__234"); - assertThrows(IllegalArgumentException.class, () -> metadataFiles.sort(TranslogTransferMetadata.METADATA_FILENAME_COMPARATOR)); + assertThrows(IOException.class, translogTransferManager::readMetadata); } public void testDownloadTranslog() throws IOException { @@ -360,6 +392,43 @@ public void testDeleteTranslogSuccess() throws Exception { verify(blobContainer).deleteBlobsIgnoringIfNotExists(eq(files)); } + public void testDeleteStaleTranslogMetadata() { + TranslogTransferManager translogTransferManager = new TranslogTransferManager( + shardId, + transferService, + remoteBaseTransferPath, + null + ); + String tm1 = new TranslogTransferMetadata(1, 1, 1, 2).getFileName(); + String tm2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName(); + String tm3 = new TranslogTransferMetadata(2, 3, 1, 2).getFileName(); + doAnswer(invocation -> { + ActionListener> actionListener = invocation.getArgument(3); + List bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(tm1, 1)); + bmList.add(new PlainBlobMetadata(tm2, 1)); + bmList.add(new PlainBlobMetadata(tm3, 1)); + actionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrderAsync(eq(ThreadPool.Names.REMOTE_PURGE), any(BlobPath.class), anyInt(), any(ActionListener.class)); + List files = List.of(tm2, tm3); + translogTransferManager.deleteStaleTranslogMetadataFilesAsync(() -> { + verify(transferService).listAllInSortedOrderAsync( + eq(ThreadPool.Names.REMOTE_PURGE), + any(BlobPath.class), + eq(Integer.MAX_VALUE), + any() + ); + verify(transferService).deleteBlobsAsync( + eq(ThreadPool.Names.REMOTE_PURGE), + any(BlobPath.class), + eq(files), + any(ActionListener.class) + ); + }); + } + public void testDeleteTranslogFailure() throws Exception { FileTransferTracker tracker = new FileTransferTracker(new ShardId("index", "indexUuid", 0)); BlobStore blobStore = mock(BlobStore.class); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 7f3819563dcbd..f5af84bb9f128 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -42,6 +42,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.Version; import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.PlainActionFuture; @@ -61,6 +62,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.fs.FsBlobContainer; @@ -68,14 +70,14 @@ import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.io.PathUtils; +import org.opensearch.common.lease.Releasable; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.common.lease.Releasable; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; @@ -98,18 +100,18 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.index.store.RemoteBufferedOutputDirectory; import org.opensearch.index.store.RemoteDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; -import org.opensearch.index.store.RemoteBufferedOutputDirectory; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.Translog; -import org.opensearch.indices.IndicesService; import org.opensearch.index.translog.TranslogFactory; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.indices.recovery.AsyncRecoveryTarget; @@ -172,6 +174,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.cluster.routing.TestShardRouting.newShardRouting; @@ -645,6 +649,17 @@ protected RepositoriesService createRepositoriesService() { when(repository.basePath()).thenReturn(new BlobPath()); BlobStore blobStore = Mockito.mock(BlobStore.class); BlobContainer blobContainer = Mockito.mock(BlobContainer.class); + doAnswer(invocation -> { + LatchedActionListener> listener = invocation.getArgument(3); + listener.onResponse(new ArrayList<>()); + return null; + }).when(blobContainer) + .listBlobsByPrefixInSortedOrder( + any(String.class), + anyInt(), + any(BlobContainer.BlobNameSortOrder.class), + any(ActionListener.class) + ); when(blobStore.blobContainer(any())).thenReturn(blobContainer); when(repository.blobStore()).thenReturn(blobStore); when(repositoriesService.repository(any(String.class))).thenReturn(repository); From e0e60eeb9a7bfdf2acd97b4eed04c0f48ada7199 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Fri, 7 Jul 2023 11:19:01 -0700 Subject: [PATCH 068/186] Bump com.google.guava:guava from 32.0.1-jre to 32.1.1-jre in /distribution/tools/upgrade-cli (#8527) * Bump com.google.guava:guava from 32.0.1-jre to 32.1.1-jre in /distribution/tools/upgrade-cli Signed-off-by: owaiskazi19 * Disabled transitive dependencies for test Signed-off-by: owaiskazi19 --------- Signed-off-by: owaiskazi19 --- CHANGELOG.md | 2 +- distribution/tools/upgrade-cli/build.gradle | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 045a1cebf4283..b0c5403253b46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) -- Bump `com.google.guava:guava` from 30.1.1-jre to 32.0.0-jre (#7565, #7811, #7807, #7808) +- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402) - Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812) - Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810) - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896) diff --git a/distribution/tools/upgrade-cli/build.gradle b/distribution/tools/upgrade-cli/build.gradle index 54d51936159ec..d81d00440a864 100644 --- a/distribution/tools/upgrade-cli/build.gradle +++ b/distribution/tools/upgrade-cli/build.gradle @@ -21,7 +21,9 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' - testRuntimeOnly 'com.google.guava:guava:32.0.1-jre' + testRuntimeOnly('com.google.guava:guava:32.1.1-jre') { + transitive = false + } } tasks.named("dependencyLicenses").configure { From 3928ddca0cbbc3d15494116a003d1f62f863f5c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:10:44 -0700 Subject: [PATCH 069/186] Bump com.google.guava:guava from 32.0.1-jre to 32.1.1-jre in /distribution/tools/plugin-cli (#8400) * Bump com.google.guava:guava in /distribution/tools/plugin-cli Bumps [com.google.guava:guava](https://github.com/google/guava) from 32.0.1-jre to 32.1.1-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] * Set transitive dependencies as false Signed-off-by: owaiskazi19 --------- Signed-off-by: dependabot[bot] Signed-off-by: owaiskazi19 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: owaiskazi19 --- CHANGELOG.md | 2 +- distribution/tools/plugin-cli/build.gradle | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0c5403253b46..4ace4b67d6a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) -- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402) +- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400) - Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812) - Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810) - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896) diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index e68fbf8d6fc3c..d39697e81914b 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -41,7 +41,9 @@ dependencies { api "org.bouncycastle:bc-fips:1.0.2.3" testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' - testRuntimeOnly 'com.google.guava:guava:32.0.1-jre' + testRuntimeOnly('com.google.guava:guava:32.1.1-jre') { + transitive = false + } implementation 'org.apache.commons:commons-compress:1.23.0' } From 6e582537098c8caceff150f4f48823ef37c29164 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 13:22:02 -0700 Subject: [PATCH 070/186] Bump com.google.guava:guava from 32.0.1-jre to 32.1.1-jre in /distribution/tools/keystore-cli (#8401) * Bump com.google.guava:guava in /distribution/tools/keystore-cli Bumps [com.google.guava:guava](https://github.com/google/guava) from 32.0.1-jre to 32.1.1-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] * Set transitive dependencies as false Signed-off-by: owaiskazi19 --------- Signed-off-by: dependabot[bot] Signed-off-by: owaiskazi19 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: owaiskazi19 --- CHANGELOG.md | 2 +- distribution/tools/keystore-cli/build.gradle | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ace4b67d6a24..843529bc1ffd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) -- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400) +- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401) - Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812) - Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810) - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896) diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle index 3005aef5b6e67..fe57b342ae298 100644 --- a/distribution/tools/keystore-cli/build.gradle +++ b/distribution/tools/keystore-cli/build.gradle @@ -35,5 +35,7 @@ dependencies { compileOnly project(":libs:opensearch-cli") testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' - testRuntimeOnly 'com.google.guava:guava:32.0.1-jre' + testRuntimeOnly('com.google.guava:guava:32.1.1-jre') { + transitive = false + } } From 31e67c17ecc6eb70d28e991b13085411e961b5bd Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Fri, 7 Jul 2023 13:53:50 -0700 Subject: [PATCH 071/186] Add early termination support for concurrent segment search (#8306) Signed-off-by: Jay Deng --- CHANGELOG.md | 1 + ...ConcurrentSegmentSearchCancellationIT.java | 26 +++++++++++ .../ConcurrentSegmentSearchTimeoutIT.java | 27 ++++++++++++ .../opensearch/search/SearchTimeoutIT.java | 29 ++++++++++--- .../search/internal/ContextIndexSearcher.java | 43 ++++++++----------- .../search/internal/SearchContext.java | 10 +++++ .../query/ConcurrentQueryPhaseSearcher.java | 32 +++++++++++--- .../opensearch/search/query/QueryPhase.java | 4 +- .../search/SearchCancellationTests.java | 16 +++++-- .../internal/ContextIndexSearcherTests.java | 9 +++- .../profile/query/QueryProfilerTests.java | 11 ++++- .../search/query/QueryPhaseTests.java | 12 +++++- .../search/query/QueryProfilePhaseTests.java | 12 +++++- .../aggregations/AggregatorTestCase.java | 5 ++- 14 files changed, 188 insertions(+), 49 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchCancellationIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchTimeoutIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 843529bc1ffd1..e0146af42ee3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) - Add support for ignoring missing Javadoc on generated code using annotation ([#7604](https://github.com/opensearch-project/OpenSearch/pull/7604)) +- Add partial results support for concurrent segment search ([#8306](https://github.com/opensearch-project/OpenSearch/pull/8306)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 diff --git a/server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchCancellationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchCancellationIT.java new file mode 100644 index 0000000000000..3c50627e342dd --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchCancellationIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search; + +import org.opensearch.common.settings.FeatureFlagSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; + +public class ConcurrentSegmentSearchCancellationIT extends SearchCancellationIT { + @Override + protected Settings featureFlagSettings() { + Settings.Builder featureSettings = Settings.builder(); + for (Setting builtInFlag : FeatureFlagSettings.BUILT_IN_FEATURE_FLAGS) { + featureSettings.put(builtInFlag.getKey(), builtInFlag.getDefaultRaw(Settings.EMPTY)); + } + featureSettings.put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true"); + return featureSettings.build(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchTimeoutIT.java new file mode 100644 index 0000000000000..c19f762679fb0 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/ConcurrentSegmentSearchTimeoutIT.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search; + +import org.opensearch.common.settings.FeatureFlagSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; + +public class ConcurrentSegmentSearchTimeoutIT extends SearchTimeoutIT { + + @Override + protected Settings featureFlagSettings() { + Settings.Builder featureSettings = Settings.builder(); + for (Setting builtInFlag : FeatureFlagSettings.BUILT_IN_FEATURE_FLAGS) { + featureSettings.put(builtInFlag.getKey(), builtInFlag.getDefaultRaw(Settings.EMPTY)); + } + featureSettings.put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true"); + return featureSettings.build(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java index 049dcb50024ba..c7392b260319a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java @@ -51,7 +51,6 @@ import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.index.query.QueryBuilders.scriptQuery; import static org.opensearch.search.SearchTimeoutIT.ScriptedTimeoutPlugin.SCRIPT_NAME; -import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) public class SearchTimeoutIT extends OpenSearchIntegTestCase { @@ -67,17 +66,37 @@ protected Settings nodeSettings(int nodeOrdinal) { } public void testSimpleTimeout() throws Exception { - for (int i = 0; i < 32; i++) { + final int numDocs = 1000; + for (int i = 0; i < numDocs; i++) { client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); } refresh("test"); SearchResponse searchResponse = client().prepareSearch("test") - .setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS)) + .setTimeout(new TimeValue(5, TimeUnit.MILLISECONDS)) .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) .setAllowPartialSearchResults(true) .get(); - assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertTrue(searchResponse.isTimedOut()); + assertEquals(0, searchResponse.getFailedShards()); + assertTrue(numDocs > searchResponse.getHits().getTotalHits().value); + } + + public void testSimpleDoesNotTimeout() throws Exception { + final int numDocs = 10; + for (int i = 0; i < numDocs; i++) { + client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); + } + refresh("test"); + + SearchResponse searchResponse = client().prepareSearch("test") + .setTimeout(new TimeValue(10000, TimeUnit.SECONDS)) + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .setAllowPartialSearchResults(true) + .get(); + assertFalse(searchResponse.isTimedOut()); + assertEquals(0, searchResponse.getFailedShards()); + assertEquals(numDocs, searchResponse.getHits().getTotalHits().value); } public void testPartialResultsIntolerantTimeout() throws Exception { @@ -91,7 +110,7 @@ public void testPartialResultsIntolerantTimeout() throws Exception { .setAllowPartialSearchResults(false) // this line causes timeouts to report failures .get() ); - assertTrue(ex.toString().contains("Time exceeded")); + assertTrue(ex.toString().contains("QueryPhaseExecutionException[Time exceeded]")); } public static class ScriptedTimeoutPlugin extends MockScriptPlugin { diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index f1650b686f070..2f1c5475c9cf6 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -73,6 +73,7 @@ import org.opensearch.search.profile.query.ProfileWeight; import org.opensearch.search.profile.query.QueryProfiler; import org.opensearch.search.profile.query.QueryTimingType; +import org.opensearch.search.query.QueryPhase; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.MinAndMax; @@ -103,26 +104,6 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { private MutableQueryTimeout cancellable; private SearchContext searchContext; - public ContextIndexSearcher( - IndexReader reader, - Similarity similarity, - QueryCache queryCache, - QueryCachingPolicy queryCachingPolicy, - boolean wrapWithExitableDirectoryReader, - Executor executor - ) throws IOException { - this( - reader, - similarity, - queryCache, - queryCachingPolicy, - new MutableQueryTimeout(), - wrapWithExitableDirectoryReader, - executor, - null - ); - } - public ContextIndexSearcher( IndexReader reader, Similarity similarity, @@ -310,18 +291,22 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto return; } - cancellable.checkCancelled(); - weight = wrapWeight(weight); - // See please https://github.com/apache/lucene/pull/964 - collector.setWeight(weight); final LeafCollector leafCollector; try { + cancellable.checkCancelled(); + weight = wrapWeight(weight); + // See please https://github.com/apache/lucene/pull/964 + collector.setWeight(weight); leafCollector = collector.getLeafCollector(ctx); } catch (CollectionTerminatedException e) { // there is no doc of interest in this reader context // continue with the following leaf return; + } catch (QueryPhase.TimeExceededException e) { + searchContext.setSearchTimedOut(true); + return; } + // catch early terminated exception and rethrow? Bits liveDocs = ctx.reader().getLiveDocs(); BitSet liveDocsBitSet = getSparseBitSetOrNull(liveDocs); if (liveDocsBitSet == null) { @@ -332,6 +317,9 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf + } catch (QueryPhase.TimeExceededException e) { + searchContext.setSearchTimedOut(true); + return; } } } else { @@ -348,6 +336,9 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf + } catch (QueryPhase.TimeExceededException e) { + searchContext.setSearchTimedOut(true); + return; } } } @@ -492,7 +483,7 @@ private boolean canMatch(LeafReaderContext ctx) throws IOException { } private boolean canMatchSearchAfter(LeafReaderContext ctx) throws IOException { - if (searchContext != null && searchContext.request() != null && searchContext.request().source() != null) { + if (searchContext.request() != null && searchContext.request().source() != null) { // Only applied on primary sort field and primary search_after. FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(searchContext.request().source()); if (primarySortField != null) { @@ -512,7 +503,7 @@ private boolean shouldReverseLeafReaderContexts() { // This is actually beneficial for search queries to start search on latest segments first for time series workload. // That can slow down ASC order queries on timestamp workload. So to avoid that slowdown, we will reverse leaf // reader order here. - if (searchContext != null && searchContext.indexShard().isTimeSeriesDescSortOptimizationEnabled()) { + if (searchContext.indexShard().isTimeSeriesDescSortOptimizationEnabled()) { // Only reverse order for asc order sort queries if (searchContext.sort() != null && searchContext.sort().sort != null diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index 9c6a0498ec848..319a5624bbf56 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -98,6 +98,8 @@ public abstract class SearchContext implements Releasable { private final AtomicBoolean closed = new AtomicBoolean(false); private InnerHitsContext innerHitsContext; + private volatile boolean searchTimedOut; + protected SearchContext() {} public abstract void setTask(SearchShardTask task); @@ -106,6 +108,14 @@ protected SearchContext() {} public abstract boolean isCancelled(); + public boolean isSearchTimedOut() { + return this.searchTimedOut; + } + + public void setSearchTimedOut(boolean searchTimedOut) { + this.searchTimedOut = searchTimedOut; + } + @Override public final void close() { if (closed.compareAndSet(false, true)) { diff --git a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java index e3ba0eda4af55..4d3b2a124f64f 100644 --- a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java +++ b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java @@ -19,10 +19,10 @@ import org.opensearch.search.internal.SearchContext; import org.opensearch.search.profile.query.ProfileCollectorManager; import org.opensearch.search.query.QueryPhase.DefaultQueryPhaseSearcher; -import org.opensearch.search.query.QueryPhase.TimeExceededException; import java.io.IOException; import java.util.LinkedList; +import java.util.concurrent.ExecutionException; import static org.opensearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext; @@ -80,12 +80,12 @@ private static boolean searchWithCollectorManager( try { final ReduceableSearchResult result = searcher.search(query, collectorManager); result.reduce(queryResult); - } catch (EarlyTerminatingCollector.EarlyTerminationException e) { - queryResult.terminatedEarly(true); - } catch (TimeExceededException e) { + } catch (RuntimeException re) { + rethrowCauseIfPossible(re, searchContext); + } + if (searchContext.isSearchTimedOut()) { assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; if (searchContext.request().allowPartialSearchResults() == false) { - // Can't rethrow TimeExceededException because not serializable throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); } queryResult.searchTimedOut(true); @@ -101,4 +101,26 @@ private static boolean searchWithCollectorManager( public AggregationProcessor aggregationProcessor(SearchContext searchContext) { return aggregationProcessor; } + + private static void rethrowCauseIfPossible(RuntimeException re, SearchContext searchContext) throws T { + // Rethrow exception if cause is null + if (re.getCause() == null) { + throw re; + } + + // Unwrap the RuntimeException and ExecutionException from Lucene concurrent search method and rethrow + if (re.getCause() instanceof ExecutionException || re.getCause() instanceof InterruptedException) { + Throwable t = re.getCause(); + if (t.getCause() != null) { + throw (T) t.getCause(); + } + } + + // Rethrow any unexpected exception types + throw new QueryPhaseExecutionException( + searchContext.shardTarget(), + "Failed to execute concurrent segment search thread", + re.getCause() + ); + } } diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java index 069b410280d63..8418fdca2f777 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java @@ -354,10 +354,10 @@ private static boolean searchWithCollector( searcher.search(query, queryCollector); } catch (EarlyTerminatingCollector.EarlyTerminationException e) { queryResult.terminatedEarly(true); - } catch (TimeExceededException e) { + } + if (searchContext.isSearchTimedOut()) { assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; if (searchContext.request().allowPartialSearchResults() == false) { - // Can't rethrow TimeExceededException because not serializable throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); } queryResult.searchTimedOut(true); diff --git a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java index db807e31d0238..e67123bf2c51e 100644 --- a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java @@ -49,7 +49,9 @@ import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.index.shard.IndexShard; import org.opensearch.search.internal.ContextIndexSearcher; +import org.opensearch.search.internal.SearchContext; import org.opensearch.tasks.TaskCancelledException; import org.opensearch.test.OpenSearchTestCase; import org.junit.AfterClass; @@ -59,6 +61,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SearchCancellationTests extends OpenSearchTestCase { @@ -109,7 +113,8 @@ public void testAddingCancellationActions() throws IOException { IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - null + null, + mock(SearchContext.class) ); NullPointerException npe = expectThrows(NullPointerException.class, () -> searcher.addQueryCancellation(null)); assertEquals("cancellation runnable should not be null", npe.getMessage()); @@ -123,13 +128,17 @@ public void testAddingCancellationActions() throws IOException { public void testCancellableCollector() throws IOException { TotalHitCountCollector collector1 = new TotalHitCountCollector(); Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); }; + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); ContextIndexSearcher searcher = new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - null + null, + searchContext ); searcher.search(new MatchAllDocsQuery(), collector1); @@ -157,7 +166,8 @@ public void testExitableDirectoryReader() throws IOException { IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - null + null, + mock(SearchContext.class) ); searcher.addQueryCancellation(cancellation); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton()); diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index 82dcbd6f69aa4..c9755cec2a33d 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -80,6 +80,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.test.OpenSearchTestCase; @@ -91,6 +92,8 @@ import java.util.IdentityHashMap; import java.util.Set; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.opensearch.search.internal.ContextIndexSearcher.intersectScorerAndBitSet; import static org.opensearch.search.internal.ExitableDirectoryReader.ExitableLeafReader; import static org.opensearch.search.internal.ExitableDirectoryReader.ExitablePointValues; @@ -253,13 +256,17 @@ public void onRemoval(ShardId shardId, Accountable accountable) { DocumentSubsetDirectoryReader filteredReader = new DocumentSubsetDirectoryReader(reader, cache, roleQuery); + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); ContextIndexSearcher searcher = new ContextIndexSearcher( filteredReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - null + null, + searchContext ); for (LeafReaderContext context : searcher.getIndexReader().leaves()) { diff --git a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java index 8a8700d853ad7..f0f692e15f066 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java @@ -60,7 +60,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.TestUtil; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.index.shard.IndexShard; import org.opensearch.search.internal.ContextIndexSearcher; +import org.opensearch.search.internal.SearchContext; import org.opensearch.search.profile.ProfileResult; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; @@ -78,6 +80,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class QueryProfilerTests extends OpenSearchTestCase { private Directory dir; @@ -112,13 +116,18 @@ public void setUp() throws Exception { } reader = w.getReader(); w.close(); + + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); searcher = new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), ALWAYS_CACHE_POLICY, true, - executor + executor, + searchContext ); } diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index dcec4842fc81e..cf4d7798056c4 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -1204,25 +1204,33 @@ public boolean lowLevelCancellation() { } private static ContextIndexSearcher newContextSearcher(IndexReader reader, ExecutorService executor) throws IOException { + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - executor + executor, + searchContext ); } private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size, ExecutorService executor) throws IOException { + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - executor + executor, + searchContext ) { @Override diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index b1fc5f1354039..9c27783263c5a 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1424,25 +1424,33 @@ private final void assertProfileData( } private static ContextIndexSearcher newContextSearcher(IndexReader reader, ExecutorService executor) throws IOException { + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - executor + executor, + searchContext ); } private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size, ExecutorService executor) throws IOException { + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true, - executor + executor, + searchContext ) { @Override diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index 48eea430aca76..cf0f9dbb04f7f 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -329,16 +329,17 @@ public boolean shouldCache(Query query) { return false; } }; + SearchContext searchContext = mock(SearchContext.class); ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher( indexSearcher.getIndexReader(), indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false, - null + null, + searchContext ); - SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); when(searchContext.searcher()).thenReturn(contextIndexSearcher); when(searchContext.fetchPhase()).thenReturn(new FetchPhase(Arrays.asList(new FetchSourcePhase(), new FetchDocValuesPhase()))); From 52a5e3f6e0ca599e3193807134ea42660ecdd195 Mon Sep 17 00:00:00 2001 From: Ryan Bogan <10944539+ryanbogan@users.noreply.github.com> Date: Fri, 7 Jul 2023 15:36:31 -0700 Subject: [PATCH 072/186] Change extension integration testing to reflect new initialization sequence (#8145) Signed-off-by: Ryan Bogan --- .../testclusters/OpenSearchCluster.java | 4 +- .../gradle/testclusters/OpenSearchNode.java | 46 ++----------------- .../TestClusterConfiguration.java | 2 +- 3 files changed, 7 insertions(+), 45 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java index b61acc934d590..ffb3360e3cc55 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java @@ -206,8 +206,8 @@ public void setTestDistribution(TestDistribution distribution) { } @Override - public void extension(ExtensionsProperties extension) { - nodes.all(each -> each.extension(extension)); + public void extension(boolean extensionsEnabled) { + nodes.all(each -> each.extension(extensionsEnabled)); } @Override diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index 7b4b5d354a00b..97c97c18bb012 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -46,8 +46,6 @@ import org.opensearch.gradle.Version; import org.opensearch.gradle.VersionProperties; import org.opensearch.gradle.info.BuildParams; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; import org.gradle.api.Action; import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; @@ -94,7 +92,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Scanner; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -144,7 +141,7 @@ public class OpenSearchNode implements TestClusterConfiguration { private final Map pluginAndModuleConfigurations = new HashMap<>(); private final List> plugins = new ArrayList<>(); private final List> modules = new ArrayList<>(); - private final List extensions = new ArrayList<>(); + private boolean extensionsEnabled = false; final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this); private final LazyPropertyMap keystoreSettings = new LazyPropertyMap<>("Keystore", this); private final LazyPropertyMap keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new); @@ -346,39 +343,8 @@ public void module(String moduleProjectPath) { } @Override - public void extension(ExtensionsProperties extensions) { - this.extensions.add(extensions); - } - - public void writeExtensionFiles() { - try { - // Creates extensions.yml in the target directory - Path destination = getDistroDir().resolve("extensions").resolve("extensions.yml"); - if (!Files.exists(getDistroDir().resolve("extensions"))) { - Files.createDirectory(getDistroDir().resolve("extensions")); - } - DumperOptions dumperOptions = new DumperOptions(); - TestExtensionsList extensionsList = new TestExtensionsList(this.extensions); - dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - Yaml yaml = new Yaml(dumperOptions); - Files.write(destination, yaml.dump(extensionsList).getBytes()); - - /* - * SnakeYaml creates a Yaml file with an unnecessary line at the top with the class name - * This section of code removes that line while keeping everything else the same. - */ - - Scanner scanner = new Scanner(destination); - scanner.nextLine(); - StringBuilder extensionsString = new StringBuilder(); - while (scanner.hasNextLine()) { - extensionsString.append("\n" + scanner.nextLine()); - } - Files.write(destination, extensionsString.toString().getBytes()); - - } catch (IOException e) { - throw new UncheckedIOException("Failed to write to extensions.yml", e); - } + public void extension(boolean extensionsEnabled) { + this.extensionsEnabled = extensionsEnabled; } @Override @@ -551,10 +517,6 @@ public synchronized void start() { logToProcessStdout("installed plugins"); } - if (!extensions.isEmpty()) { - writeExtensionFiles(); - } - logToProcessStdout("Creating opensearch keystore with password set to [" + keystorePassword + "]"); if (keystorePassword.length() > 0) { runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword, "opensearch-keystore", "create", "-p"); @@ -829,7 +791,7 @@ private void startOpenSearchProcess() { environment.clear(); environment.putAll(getOpenSearchEnvironment()); - if (!extensions.isEmpty()) { + if (extensionsEnabled) { environment.put("OPENSEARCH_JAVA_OPTS", "-Dopensearch.experimental.feature.extensions.enabled=true"); } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java index 7378804d59d41..87cce2f0b32c0 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java @@ -55,7 +55,7 @@ public interface TestClusterConfiguration { void setTestDistribution(TestDistribution distribution); - void extension(ExtensionsProperties extension); + void extension(boolean extensionsEnabled); void plugin(Provider plugin); From 2bced5de0f7e36b4eb9a78487bf5c08490bb57dc Mon Sep 17 00:00:00 2001 From: Ashish Date: Sat, 8 Jul 2023 09:46:06 +0530 Subject: [PATCH 073/186] [Remote Store] Mute testRemoteRefreshRetryOnFailure test case (#8542) Signed-off-by: Ashish Singh --- .../org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java index 4005e6359a2f7..13f76adc8a5a7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java @@ -27,6 +27,7 @@ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockRepositoryIntegTestCase { + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7703") public void testRemoteRefreshRetryOnFailure() throws Exception { Path location = randomRepoPath().toAbsolutePath(); From 516685d5f8e59423552b668b3034789aa9d77961 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Sat, 8 Jul 2023 16:36:01 -0700 Subject: [PATCH 074/186] [Segment Replication] Prevent store clean up post reader close and refactor (#8463) * [Segment Replication] Prevent store clean up on reader close action Signed-off-by: Suraj Singh * [Segment Replication] Self review Signed-off-by: Suraj Singh * Address review comment Signed-off-by: Suraj Singh * Address review comments & refactor Signed-off-by: Suraj Singh * Comment Signed-off-by: Suraj Singh * Fix unit test Signed-off-by: Suraj Singh * Unit test to verify temporary files are not deleted from commits Signed-off-by: Suraj Singh * Compilation error fix Signed-off-by: Suraj Singh * Javadoc Signed-off-by: Suraj Singh * Skip testScrollWithConcurrentIndexAndSearch with remote store Signed-off-by: Suraj Singh --------- Signed-off-by: Suraj Singh --- .../replication/SegmentReplicationIT.java | 55 +++++ .../index/engine/NRTReplicationEngine.java | 20 +- .../engine/NRTReplicationReaderManager.java | 2 + .../org/opensearch/index/store/Store.java | 117 ++++++----- .../indices/recovery/MultiFileWriter.java | 4 + .../replication/SegmentReplicationTarget.java | 16 +- .../engine/NRTReplicationEngineTests.java | 33 +++ .../SegmentReplicationIndexShardTests.java | 198 +++++++++++++++++- .../opensearch/index/store/StoreTests.java | 9 +- .../index/shard/IndexShardTestCase.java | 156 ++++++++++---- 10 files changed, 480 insertions(+), 130 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index ac57c78d20b73..873c05843fb56 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -25,6 +25,7 @@ import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.CreatePitAction; import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; @@ -79,6 +80,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -272,6 +274,59 @@ public void testIndexReopenClose() throws Exception { verifyStoreContent(); } + public void testScrollWithConcurrentIndexAndSearch() throws Exception { + assumeFalse("Skipping the test with Remote store as its flaky.", segmentReplicationWithRemoteEnabled()); + final String primary = internalCluster().startDataOnlyNode(); + final String replica = internalCluster().startDataOnlyNode(); + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + final List> pendingIndexResponses = new ArrayList<>(); + final List> pendingSearchResponse = new ArrayList<>(); + final int searchCount = randomIntBetween(10, 20); + final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); + + for (int i = 0; i < searchCount; i++) { + pendingIndexResponses.add( + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(i)) + .setRefreshPolicy(refreshPolicy) + .setSource("field", "value" + i) + .execute() + ); + flush(INDEX_NAME); + forceMerge(); + } + + final SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setIndices(INDEX_NAME) + .setRequestCache(false) + .setScroll(TimeValue.timeValueDays(1)) + .setSize(10) + .get(); + + for (int i = searchCount; i < searchCount * 2; i++) { + pendingIndexResponses.add( + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(i)) + .setRefreshPolicy(refreshPolicy) + .setSource("field", "value" + i) + .execute() + ); + } + flush(INDEX_NAME); + forceMerge(); + client().prepareClearScroll().addScrollId(searchResponse.getScrollId()).get(); + + assertBusy(() -> { + client().admin().indices().prepareRefresh().execute().actionGet(); + assertTrue(pendingIndexResponses.stream().allMatch(ActionFuture::isDone)); + assertTrue(pendingSearchResponse.stream().allMatch(ActionFuture::isDone)); + }, 1, TimeUnit.MINUTES); + verifyStoreContent(); + waitForSearchableDocs(INDEX_NAME, 2 * searchCount, List.of(primary, replica)); + } + public void testMultipleShards() throws Exception { Settings indexSettings = Settings.builder() .put(super.indexSettings()) diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index 50b5fbb8596a6..b5552ed552f09 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -123,21 +123,7 @@ private NRTReplicationReaderManager buildReaderManager() throws IOException { return new NRTReplicationReaderManager( OpenSearchDirectoryReader.wrap(getDirectoryReader(), shardId), store::incRefFileDeleter, - (files) -> { - store.decRefFileDeleter(files); - try { - store.cleanupAndPreserveLatestCommitPoint( - "On reader closed", - getLatestSegmentInfos(), - getLastCommittedSegmentInfos(), - false - ); - } catch (IOException e) { - // Log but do not rethrow - we can try cleaning up again after next replication cycle. - // If that were to fail, the shard will as well. - logger.error("Unable to clean store after reader closed", e); - } - } + store::decRefFileDeleter ); } @@ -147,9 +133,9 @@ public TranslogManager translogManager() { } public synchronized void updateSegments(final SegmentInfos infos) throws IOException { - // Update the current infos reference on the Engine's reader. - ensureOpen(); try (ReleasableLock lock = writeLock.acquire()) { + // Update the current infos reference on the Engine's reader. + ensureOpen(); final long maxSeqNo = Long.parseLong(infos.userData.get(MAX_SEQ_NO)); final long incomingGeneration = infos.getGeneration(); readerManager.updateSegments(infos); diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java index 9ec484ebfd383..268ba1a436393 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java @@ -72,6 +72,8 @@ protected OpenSearchDirectoryReader refreshIfNeeded(OpenSearchDirectoryReader re for (LeafReaderContext ctx : standardDirectoryReader.leaves()) { subs.add(ctx.reader()); } + // Segment_n here is ignored because it is either already committed on disk as part of previous commit point or + // does not yet exist on store (not yet committed) final Collection files = currentInfos.files(false); DirectoryReader innerReader = StandardDirectoryReader.open(referenceToRefresh.directory(), currentInfos, subs, null); final DirectoryReader softDeletesDirectoryReaderWrapper = new SoftDeletesDirectoryReaderWrapper( diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 90832b4c77756..46e5e627ef415 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -50,6 +50,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedChecksum; +import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; @@ -64,7 +65,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.Version; import org.opensearch.ExceptionsHelper; -import org.opensearch.common.Nullable; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.UUIDs; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -789,69 +790,33 @@ public void cleanupAndVerify(String reason, MetadataSnapshot sourceMetadata) thr /** * Segment Replication method - * This method deletes every file in this store that is not referenced by the passed in SegmentInfos or - * part of the latest on-disk commit point. + * This method deletes files in store that are not referenced by latest on-disk commit point * - * This method is used for segment replication when the in memory SegmentInfos can be ahead of the on disk segment file. - * In this case files from both snapshots must be preserved. Verification has been done that all files are present on disk. * @param reason the reason for this cleanup operation logged for each deleted file - * @param infos {@link SegmentInfos} Files from this infos will be preserved on disk if present. - * @throws IllegalStateException if the latest snapshot in this store differs from the given one after the cleanup. - */ - public void cleanupAndPreserveLatestCommitPoint(String reason, SegmentInfos infos) throws IOException { - this.cleanupAndPreserveLatestCommitPoint(reason, infos, readLastCommittedSegmentsInfo(), true); - } - - /** - * Segment Replication method - * - * Similar to {@link Store#cleanupAndPreserveLatestCommitPoint(String, SegmentInfos)} with extra parameters for cleanup - * - * This method deletes every file in this store. Except - * 1. Files referenced by the passed in SegmentInfos, usually in-memory segment infos copied from primary - * 2. Files part of the passed in segment infos, typically the last committed segment info - * 3. Files incremented by active reader for pit/scroll queries - * 4. Temporary replication file if passed in deleteTempFiles is true. - * - * @param reason the reason for this cleanup operation logged for each deleted file - * @param infos {@link SegmentInfos} Files from this infos will be preserved on disk if present. - * @param lastCommittedSegmentInfos {@link SegmentInfos} Last committed segment infos - * @param deleteTempFiles Does this clean up delete temporary replication files + * @param fileToConsiderForCleanUp Files to consider for clean up. * - * @throws IllegalStateException if the latest snapshot in this store differs from the given one after the cleanup. + * @throws IOException Exception on locking. */ - public void cleanupAndPreserveLatestCommitPoint( - String reason, - SegmentInfos infos, - SegmentInfos lastCommittedSegmentInfos, - boolean deleteTempFiles - ) throws IOException { + public void cleanupAndPreserveLatestCommitPoint(Collection fileToConsiderForCleanUp, String reason) throws IOException { assert indexSettings.isSegRepEnabled(); // fetch a snapshot from the latest on disk Segments_N file. This can be behind // the passed in local in memory snapshot, so we want to ensure files it references are not removed. metadataLock.writeLock().lock(); try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { - cleanupFiles(reason, lastCommittedSegmentInfos.files(true), infos.files(true), deleteTempFiles); + cleanupFiles(fileToConsiderForCleanUp, reason, this.readLastCommittedSegmentsInfo().files(true)); } finally { metadataLock.writeLock().unlock(); } } - private void cleanupFiles( - String reason, - Collection localSnapshot, - @Nullable Collection additionalFiles, - boolean deleteTempFiles - ) throws IOException { + private void cleanupFiles(Collection filesToConsiderForCleanup, String reason, Collection lastCommittedSegmentInfos) { assert metadataLock.isWriteLockedByCurrentThread(); - for (String existingFile : directory.listAll()) { - if (Store.isAutogenerated(existingFile) - || localSnapshot != null && localSnapshot.contains(existingFile) - || (additionalFiles != null && additionalFiles.contains(existingFile)) - // also ensure we are not deleting a file referenced by an active reader. + for (String existingFile : filesToConsiderForCleanup) { + if (Store.isAutogenerated(existingFile) || lastCommittedSegmentInfos != null && lastCommittedSegmentInfos.contains(existingFile) + // also ensure we are not deleting a file referenced by an active reader. || replicaFileTracker != null && replicaFileTracker.canDelete(existingFile) == false - // prevent temporary file deletion during reader cleanup - || deleteTempFiles == false && existingFile.startsWith(REPLICATION_PREFIX)) { + // Prevent temporary replication files as it should be cleaned up MultiFileWriter + || existingFile.startsWith(REPLICATION_PREFIX)) { // don't delete snapshot file, or the checksums file (note, this is extra protection since the Store won't delete // checksum) continue; @@ -871,6 +836,53 @@ private void cleanupFiles( } } + /** + * Segment replication method + * + * This method takes the segment info bytes to build SegmentInfos. It inc'refs files pointed by passed in SegmentInfos + * bytes to ensure they are not deleted. + * + * @param tmpToFileName Map of temporary replication file to actual file name + * @param infosBytes bytes[] of SegmentInfos supposed to be sent over by primary excluding segment_N file + * @param segmentsGen segment generation number + * @param consumer consumer for generated SegmentInfos + * @throws IOException Exception while reading store and building segment infos + */ + public void buildInfosFromBytes( + Map tmpToFileName, + byte[] infosBytes, + long segmentsGen, + CheckedConsumer consumer + ) throws IOException { + metadataLock.writeLock().lock(); + try { + final List values = new ArrayList<>(tmpToFileName.values()); + incRefFileDeleter(values); + try { + renameTempFilesSafe(tmpToFileName); + consumer.accept(buildSegmentInfos(infosBytes, segmentsGen)); + } finally { + decRefFileDeleter(values); + } + } finally { + metadataLock.writeLock().unlock(); + } + } + + private SegmentInfos buildSegmentInfos(byte[] infosBytes, long segmentsGen) throws IOException { + try (final ChecksumIndexInput input = toIndexInput(infosBytes)) { + return SegmentInfos.readCommit(directory, input, segmentsGen); + } + } + + /** + * This method formats byte[] containing the primary's SegmentInfos into lucene's {@link ChecksumIndexInput} that can be + * passed to SegmentInfos.readCommit + */ + private ChecksumIndexInput toIndexInput(byte[] input) { + return new BufferedChecksumIndexInput(new ByteArrayIndexInput("Snapshot of SegmentInfos", input)); + } + // pkg private for testing final void verifyAfterCleanup(MetadataSnapshot sourceMetadata, MetadataSnapshot targetMetadata) { final RecoveryDiff recoveryDiff = targetMetadata.recoveryDiff(sourceMetadata); @@ -945,7 +957,7 @@ public void commitSegmentInfos(SegmentInfos latestSegmentInfos, long maxSeqNo, l latestSegmentInfos.commit(directory()); directory.sync(latestSegmentInfos.files(true)); directory.syncMetaData(); - cleanupAndPreserveLatestCommitPoint("After commit", latestSegmentInfos); + cleanupAndPreserveLatestCommitPoint(List.of(this.directory.listAll()), "After commit"); } finally { metadataLock.writeLock().unlock(); } @@ -1961,6 +1973,13 @@ public void incRefFileDeleter(Collection files) { public void decRefFileDeleter(Collection files) { if (this.indexSettings.isSegRepEnabled()) { this.replicaFileTracker.decRef(files); + try { + this.cleanupAndPreserveLatestCommitPoint(files, "On reader close"); + } catch (IOException e) { + // Log but do not rethrow - we can try cleaning up again after next replication cycle. + // If that were to fail, the shard will as well. + logger.error("Unable to clean store after reader closed", e); + } } } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java b/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java index 4f9db27ffc9db..c27852b27960b 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java +++ b/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java @@ -97,6 +97,10 @@ String getTempNameForFile(String origFile) { return tempFilePrefix + origFile; } + public Map getTempFileNames() { + return tempFileNames; + } + public IndexOutput getOpenIndexOutput(String key) { ensureOpen.run(); return openIndexOutputs.get(key); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 22c68ad46fea6..9d724d6cc9dcf 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -21,6 +21,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.UUIDs; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; @@ -221,18 +222,15 @@ private void finalizeReplication(CheckpointInfoResponse checkpointInfoResponse, state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); Store store = null; try { - multiFileWriter.renameAllTempFiles(); store = store(); store.incRef(); - // Deserialize the new SegmentInfos object sent from the primary. - final ReplicationCheckpoint responseCheckpoint = checkpointInfoResponse.getCheckpoint(); - SegmentInfos infos = SegmentInfos.readCommit( - store.directory(), - toIndexInput(checkpointInfoResponse.getInfosBytes()), - responseCheckpoint.getSegmentsGen() + CheckedConsumer finalizeReplication = indexShard::finalizeReplication; + store.buildInfosFromBytes( + multiFileWriter.getTempFileNames(), + checkpointInfoResponse.getInfosBytes(), + checkpointInfoResponse.getCheckpoint().getSegmentsGen(), + finalizeReplication ); - cancellableThreads.checkForCancel(); - indexShard.finalizeReplication(infos); } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) { // this is a fatal exception at this stage. // this means we transferred files from the remote that have not be checksummed and they are diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java index 3e1112ae3069b..e25e6ea206d84 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -160,6 +161,38 @@ public void testUpdateSegments_replicaReceivesSISWithLowerGen() throws IOExcepti } } + public void testSimultaneousEngineCloseAndCommit() throws IOException, InterruptedException { + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + try ( + final Store nrtEngineStore = createStore(INDEX_SETTINGS, newDirectory()); + final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore) + ) { + CountDownLatch latch = new CountDownLatch(1); + Thread commitThread = new Thread(() -> { + try { + nrtEngine.updateSegments(store.readLastCommittedSegmentsInfo()); + latch.countDown(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + Thread closeThread = new Thread(() -> { + try { + latch.await(); + nrtEngine.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + commitThread.start(); + closeThread.start(); + commitThread.join(); + closeThread.join(); + } + } + public void testUpdateSegments_replicaCommitsFirstReceivedInfos() throws IOException { final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 0c859c5f6a64a..b3876a8ea8fd0 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -13,13 +13,14 @@ import org.junit.Assert; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; +import org.opensearch.action.admin.indices.flush.FlushRequest; +import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingHelper; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.settings.ClusterSettings; @@ -30,6 +31,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.DocIdSeqNoAndSource; +import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.InternalEngine; import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.engine.NRTReplicationEngine; @@ -63,6 +65,8 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -70,7 +74,10 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Arrays.asList; import static org.hamcrest.Matchers.containsString; @@ -78,6 +85,7 @@ import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -169,6 +177,186 @@ public void testIsSegmentReplicationAllowed_WrongEngineType() throws IOException closeShards(indexShard); } + /** + * This test mimics the segment replication failure due to CorruptIndexException exception which happens when + * reader close operation on replica shard deletes the segment files copied in current round of segment replication. + * It does this by blocking the finalizeReplication on replica shard and performing close operation on acquired + * searcher that triggers the reader close operation. + * @throws Exception + */ + public void testSegmentReplication_With_ReaderClosedConcurrently() throws Exception { + String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; + try (ReplicationGroup shards = createGroup(1, settings, mappings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard primaryShard = shards.getPrimary(); + final IndexShard replicaShard = shards.getReplicas().get(0); + + // Step 1. Ingest numDocs documents & replicate to replica shard + final int numDocs = randomIntBetween(100, 200); + logger.info("--> Inserting documents {}", numDocs); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + assertEqualTranslogOperations(shards, primaryShard); + primaryShard.refresh("Test"); + primaryShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); + replicateSegments(primaryShard, shards.getReplicas()); + + IndexShard spyShard = spy(replicaShard); + Engine.Searcher test = replicaShard.getEngine().acquireSearcher("testSegmentReplication_With_ReaderClosedConcurrently"); + shards.assertAllEqual(numDocs); + + // Step 2. Ingest numDocs documents again & replicate to replica shard + logger.info("--> Ingest {} docs again", numDocs); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + assertEqualTranslogOperations(shards, primaryShard); + primaryShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); + replicateSegments(primaryShard, shards.getReplicas()); + + // Step 3. Perform force merge down to 1 segment on primary + primaryShard.forceMerge(new ForceMergeRequest().maxNumSegments(1).flush(true)); + logger.info("--> primary store after force merge {}", Arrays.toString(primaryShard.store().directory().listAll())); + // Perform close on searcher before IndexShard::finalizeReplication + doAnswer(n -> { + test.close(); + n.callRealMethod(); + return null; + }).when(spyShard).finalizeReplication(any()); + replicateSegments(primaryShard, List.of(spyShard)); + shards.assertAllEqual(numDocs); + } + } + + /** + * Similar to test above, this test shows the issue where an engine close operation during active segment replication + * can result in Lucene CorruptIndexException. + * @throws Exception + */ + public void testSegmentReplication_With_EngineClosedConcurrently() throws Exception { + String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; + try (ReplicationGroup shards = createGroup(1, settings, mappings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard primaryShard = shards.getPrimary(); + final IndexShard replicaShard = shards.getReplicas().get(0); + + // Step 1. Ingest numDocs documents + final int numDocs = randomIntBetween(100, 200); + logger.info("--> Inserting documents {}", numDocs); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + assertEqualTranslogOperations(shards, primaryShard); + primaryShard.refresh("Test"); + primaryShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); + replicateSegments(primaryShard, shards.getReplicas()); + shards.assertAllEqual(numDocs); + + // Step 2. Ingest numDocs documents again to create a new commit + logger.info("--> Ingest {} docs again", numDocs); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + assertEqualTranslogOperations(shards, primaryShard); + primaryShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); + logger.info("--> primary store after final flush {}", Arrays.toString(primaryShard.store().directory().listAll())); + + // Step 3. Before replicating segments, block finalizeReplication and perform engine commit directly that + // cleans up recently copied over files + IndexShard spyShard = spy(replicaShard); + doAnswer(n -> { + NRTReplicationEngine engine = (NRTReplicationEngine) replicaShard.getEngine(); + // Using engine.close() prevents indexShard.finalizeReplication execution due to engine AlreadyClosedException, + // thus as workaround, use updateSegments which eventually calls commitSegmentInfos on latest segment infos. + engine.updateSegments(engine.getSegmentInfosSnapshot().get()); + n.callRealMethod(); + return null; + }).when(spyShard).finalizeReplication(any()); + replicateSegments(primaryShard, List.of(spyShard)); + shards.assertAllEqual(numDocs); + } + } + + /** + * Verifies that commits on replica engine resulting from engine or reader close does not cleanup the temporary + * replication files from ongoing round of segment replication + * @throws Exception + */ + public void testTemporaryFilesNotCleanup() throws Exception { + String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; + try (ReplicationGroup shards = createGroup(1, settings, mappings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard primaryShard = shards.getPrimary(); + final IndexShard replica = shards.getReplicas().get(0); + + // Step 1. Ingest numDocs documents, commit to create commit point on primary & replicate + final int numDocs = randomIntBetween(100, 200); + logger.info("--> Inserting documents {}", numDocs); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + assertEqualTranslogOperations(shards, primaryShard); + primaryShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); + replicateSegments(primaryShard, shards.getReplicas()); + shards.assertAllEqual(numDocs); + + // Step 2. Ingest numDocs documents again to create a new commit on primary + logger.info("--> Ingest {} docs again", numDocs); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + assertEqualTranslogOperations(shards, primaryShard); + primaryShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); + + // Step 3. Copy segment files to replica shard but prevent commit + final CountDownLatch countDownLatch = new CountDownLatch(1); + Map primaryMetadata; + try (final GatedCloseable segmentInfosSnapshot = primaryShard.getSegmentInfosSnapshot()) { + final SegmentInfos primarySegmentInfos = segmentInfosSnapshot.get(); + primaryMetadata = primaryShard.store().getSegmentMetadataMap(primarySegmentInfos); + } + final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); + final IndicesService indicesService = mock(IndicesService.class); + when(indicesService.getShardOrNull(replica.shardId)).thenReturn(replica); + final SegmentReplicationTargetService targetService = new SegmentReplicationTargetService( + threadPool, + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + mock(TransportService.class), + sourceFactory, + indicesService, + clusterService + ); + final Consumer runnablePostGetFiles = (indexShard) -> { + try { + Collection temporaryFiles = Stream.of(indexShard.store().directory().listAll()) + .filter(name -> name.startsWith(SegmentReplicationTarget.REPLICATION_PREFIX)) + .collect(Collectors.toList()); + + // Step 4. Perform a commit on replica shard. + NRTReplicationEngine engine = (NRTReplicationEngine) indexShard.getEngine(); + engine.updateSegments(engine.getSegmentInfosSnapshot().get()); + + // Step 5. Validate temporary files are not deleted from store. + Collection replicaStoreFiles = List.of(indexShard.store().directory().listAll()); + assertTrue(replicaStoreFiles.containsAll(temporaryFiles)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + SegmentReplicationSource segmentReplicationSource = getSegmentReplicationSource( + primaryShard, + (repId) -> targetService.get(repId), + runnablePostGetFiles + ); + when(sourceFactory.get(any())).thenReturn(segmentReplicationSource); + targetService.startReplication(replica, getTargetListener(primaryShard, replica, primaryMetadata, countDownLatch)); + countDownLatch.await(30, TimeUnit.SECONDS); + assertEquals("Replication failed", 0, countDownLatch.getCount()); + shards.assertAllEqual(numDocs); + } + } + public void testSegmentReplication_Index_Update_Delete() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; try (ReplicationGroup shards = createGroup(2, settings, mappings, new NRTReplicationEngineFactory())) { @@ -296,13 +484,7 @@ public void testPublishCheckpointAfterRelocationHandOff() throws IOException { public void testRejectCheckpointOnShardRoutingPrimary() throws IOException { IndexShard primaryShard = newStartedShard(true); SegmentReplicationTargetService sut; - sut = prepareForReplication( - primaryShard, - null, - mock(TransportService.class), - mock(IndicesService.class), - mock(ClusterService.class) - ); + sut = prepareForReplication(primaryShard, null); SegmentReplicationTargetService spy = spy(sut); // Starting a new shard in PrimaryMode and shard routing primary. diff --git a/server/src/test/java/org/opensearch/index/store/StoreTests.java b/server/src/test/java/org/opensearch/index/store/StoreTests.java index b11e8554027b1..d9493e63fd34c 100644 --- a/server/src/test/java/org/opensearch/index/store/StoreTests.java +++ b/server/src/test/java/org/opensearch/index/store/StoreTests.java @@ -100,6 +100,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -108,6 +109,8 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.unmodifiableMap; import static org.hamcrest.Matchers.anyOf; @@ -1191,8 +1194,12 @@ public void testCleanupAndPreserveLatestCommitPoint() throws IOException { } assertFalse(additionalSegments.isEmpty()); + Collection filesToConsiderForCleanUp = Stream.of(store.readLastCommittedSegmentsInfo().files(true), additionalSegments) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + // clean up everything not in the latest commit point. - store.cleanupAndPreserveLatestCommitPoint("test", store.readLastCommittedSegmentsInfo()); + store.cleanupAndPreserveLatestCommitPoint(filesToConsiderForCleanUp, "test"); // we want to ensure commitMetadata files are preserved after calling cleanup for (String existingFile : store.directory().listAll()) { diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index f5af84bb9f128..984605007dad7 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -1330,15 +1330,22 @@ public static Engine.Warmer createTestWarmer(IndexSettings indexSettings) { * Segment Replication specific test method - Creates a {@link SegmentReplicationTargetService} to perform replications that has * been configured to return the given primaryShard's current segments. * - * @param primaryShard {@link IndexShard} - The primary shard to replicate from. - * @param target {@link IndexShard} - The target replica shard in segment replication. + * @param primaryShard {@link IndexShard} - The target replica shard in segment replication. + * @param target {@link IndexShard} - The source primary shard in segment replication. + * @param transportService {@link TransportService} - Transport service to be used on target + * @param indicesService {@link IndicesService} - The indices service to be used on target + * @param clusterService {@link ClusterService} - The cluster service to be used on target + * @param postGetFilesRunnable - Consumer which is executed after file copy operation. This can be used to stub operations + * which are desired right after files are copied. e.g. To work with temp files + * @return Returns SegmentReplicationTargetService */ public final SegmentReplicationTargetService prepareForReplication( IndexShard primaryShard, IndexShard target, TransportService transportService, IndicesService indicesService, - ClusterService clusterService + ClusterService clusterService, + Consumer postGetFilesRunnable ) { final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); final SegmentReplicationTargetService targetService = new SegmentReplicationTargetService( @@ -1349,7 +1356,102 @@ public final SegmentReplicationTargetService prepareForReplication( indicesService, clusterService ); - final SegmentReplicationSource replicationSource = new TestReplicationSource() { + final SegmentReplicationSource replicationSource = getSegmentReplicationSource( + primaryShard, + (repId) -> targetService.get(repId), + postGetFilesRunnable + ); + when(sourceFactory.get(any())).thenReturn(replicationSource); + when(indicesService.getShardOrNull(any())).thenReturn(target); + return targetService; + } + + /** + * Segment Replication specific test method - Creates a {@link SegmentReplicationTargetService} to perform replications that has + * been configured to return the given primaryShard's current segments. + * + * @param primaryShard {@link IndexShard} - The primary shard to replicate from. + * @param target {@link IndexShard} - The target replica shard. + * @return Returns SegmentReplicationTargetService + */ + public final SegmentReplicationTargetService prepareForReplication(IndexShard primaryShard, IndexShard target) { + return prepareForReplication( + primaryShard, + target, + mock(TransportService.class), + mock(IndicesService.class), + mock(ClusterService.class), + (indexShard) -> {} + ); + } + + public final SegmentReplicationTargetService prepareForReplication( + IndexShard primaryShard, + IndexShard target, + TransportService transportService, + IndicesService indicesService, + ClusterService clusterService + ) { + return prepareForReplication(primaryShard, target, transportService, indicesService, clusterService, (indexShard) -> {}); + } + + /** + * Get listener on started segment replication event which verifies replica shard store with primary's after completion + * @param primaryShard - source of segment replication + * @param replicaShard - target of segment replication + * @param primaryMetadata - primary shard metadata before start of segment replication + * @param latch - Latch which allows consumers of this utility to ensure segment replication completed successfully + * @return Returns SegmentReplicationTargetService.SegmentReplicationListener + */ + public SegmentReplicationTargetService.SegmentReplicationListener getTargetListener( + IndexShard primaryShard, + IndexShard replicaShard, + Map primaryMetadata, + CountDownLatch latch + ) { + return new SegmentReplicationTargetService.SegmentReplicationListener() { + @Override + public void onReplicationDone(SegmentReplicationState state) { + try (final GatedCloseable snapshot = replicaShard.getSegmentInfosSnapshot()) { + final SegmentInfos replicaInfos = snapshot.get(); + final Map replicaMetadata = replicaShard.store().getSegmentMetadataMap(replicaInfos); + final Store.RecoveryDiff recoveryDiff = Store.segmentReplicationDiff(primaryMetadata, replicaMetadata); + assertTrue(recoveryDiff.missing.isEmpty()); + assertTrue(recoveryDiff.different.isEmpty()); + assertEquals(recoveryDiff.identical.size(), primaryMetadata.size()); + primaryShard.updateVisibleCheckpointForShard( + replicaShard.routingEntry().allocationId().getId(), + primaryShard.getLatestReplicationCheckpoint() + ); + } catch (Exception e) { + throw ExceptionsHelper.convertToRuntime(e); + } finally { + latch.countDown(); + } + } + + @Override + public void onReplicationFailure(SegmentReplicationState state, ReplicationFailedException e, boolean sendShardFailure) { + logger.error("Unexpected replication failure in test", e); + Assert.fail("test replication should not fail: " + e); + } + }; + } + + /** + * Utility method which creates a segment replication source, which copies files from primary shard to target shard + * @param primaryShard Primary IndexShard - source of segment replication + * @param getTargetFunc - provides replication target from target service using replication id + * @param postGetFilesRunnable - Consumer which is executed after file copy operation. This can be used to stub operations + * which are desired right after files are copied. e.g. To work with temp files + * @return Return SegmentReplicationSource + */ + public SegmentReplicationSource getSegmentReplicationSource( + IndexShard primaryShard, + Function> getTargetFunc, + Consumer postGetFilesRunnable + ) { + return new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -1380,18 +1482,16 @@ public void getSegmentFiles( ActionListener listener ) { try ( - final ReplicationCollection.ReplicationRef replicationRef = targetService.get(replicationId) + final ReplicationCollection.ReplicationRef replicationRef = getTargetFunc.apply(replicationId) ) { writeFileChunks(replicationRef.get(), primaryShard, filesToFetch.toArray(new StoreFileMetadata[] {})); } catch (IOException e) { listener.onFailure(e); } + postGetFilesRunnable.accept(indexShard); listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } }; - when(sourceFactory.get(any())).thenReturn(replicationSource); - when(indicesService.getShardOrNull(any())).thenReturn(target); - return targetService; } /** @@ -1412,46 +1512,10 @@ public final List replicateSegments(IndexShard primary } List ids = new ArrayList<>(); for (IndexShard replica : replicaShards) { - final SegmentReplicationTargetService targetService = prepareForReplication( - primaryShard, - replica, - mock(TransportService.class), - mock(IndicesService.class), - mock(ClusterService.class) - ); + final SegmentReplicationTargetService targetService = prepareForReplication(primaryShard, replica); final SegmentReplicationTarget target = targetService.startReplication( replica, - new SegmentReplicationTargetService.SegmentReplicationListener() { - @Override - public void onReplicationDone(SegmentReplicationState state) { - try (final GatedCloseable snapshot = replica.getSegmentInfosSnapshot()) { - final SegmentInfos replicaInfos = snapshot.get(); - final Map replicaMetadata = replica.store().getSegmentMetadataMap(replicaInfos); - final Store.RecoveryDiff recoveryDiff = Store.segmentReplicationDiff(primaryMetadata, replicaMetadata); - assertTrue(recoveryDiff.missing.isEmpty()); - assertTrue(recoveryDiff.different.isEmpty()); - assertEquals(recoveryDiff.identical.size(), primaryMetadata.size()); - primaryShard.updateVisibleCheckpointForShard( - replica.routingEntry().allocationId().getId(), - primaryShard.getLatestReplicationCheckpoint() - ); - } catch (Exception e) { - throw ExceptionsHelper.convertToRuntime(e); - } finally { - countDownLatch.countDown(); - } - } - - @Override - public void onReplicationFailure( - SegmentReplicationState state, - ReplicationFailedException e, - boolean sendShardFailure - ) { - logger.error("Unexpected replication failure in test", e); - Assert.fail("test replication should not fail: " + e); - } - } + getTargetListener(primaryShard, replica, primaryMetadata, countDownLatch) ); ids.add(target); } From fa3412131280f2e3943a08f06d82aaaccc063df4 Mon Sep 17 00:00:00 2001 From: Ashish Date: Sun, 9 Jul 2023 08:38:00 +0530 Subject: [PATCH 075/186] Fix flakyness in RemoteStoreRefreshListenerIT (#8547) --------- Signed-off-by: Ashish Singh --- .../remotestore/RemoteStoreRefreshListenerIT.java | 1 - .../index/shard/RemoteStoreRefreshListener.java | 14 ++++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java index 13f76adc8a5a7..4005e6359a2f7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java @@ -27,7 +27,6 @@ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockRepositoryIntegTestCase { - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7703") public void testRemoteRefreshRetryOnFailure() throws Exception { Path location = randomRepoPath().toAbsolutePath(); diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index cd3e7aa3b11a9..aaba74cd54341 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -47,7 +47,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -96,9 +95,9 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres private long primaryTerm; /** - * Semaphore that ensures there is only 1 retry scheduled at any time. + * This boolean is used to ensure that there is only 1 retry scheduled/running at any time. */ - private final Semaphore SCHEDULE_RETRY_PERMITS = new Semaphore(1); + private final AtomicBoolean retryScheduled = new AtomicBoolean(false); private volatile Iterator backoffDelayIterator; @@ -321,6 +320,9 @@ private void onSuccessfulSegmentsSync( private void cancelAndResetScheduledCancellableRetry() { if (scheduledCancellableRetry != null && scheduledCancellableRetry.getDelay(TimeUnit.NANOSECONDS) > 0) { scheduledCancellableRetry.cancel(); + // Since we are cancelling the retry attempt as an internal/external refresh happened already before the retry job could be + // started and the current run successfully uploaded the segments. + retryScheduled.set(false); } scheduledCancellableRetry = null; } @@ -333,14 +335,14 @@ private void resetBackOffDelayIterator() { } private void afterSegmentsSync(boolean isRetry, boolean shouldRetry) { - // If this was a retry attempt, then we release the semaphore at the end so that further retries can be scheduled + // If this was a retry attempt, then we set the retryScheduled to false so that the next retry (if needed) can be scheduled if (isRetry) { - SCHEDULE_RETRY_PERMITS.release(); + retryScheduled.set(false); } // If there are failures in uploading segments, then we should retry as search idle can lead to // refresh not occurring until write happens. - if (shouldRetry && indexShard.state() != IndexShardState.CLOSED && SCHEDULE_RETRY_PERMITS.tryAcquire()) { + if (shouldRetry && indexShard.state() != IndexShardState.CLOSED && retryScheduled.compareAndSet(false, true)) { scheduledCancellableRetry = indexShard.getThreadPool() .schedule(() -> this.syncSegments(true), backoffDelayIterator.next(), ThreadPool.Names.REMOTE_REFRESH); } From 9fd1ddb6a2c2df60a9b201a6b5e47041fa145ecd Mon Sep 17 00:00:00 2001 From: Ashish Date: Mon, 10 Jul 2023 10:31:15 +0530 Subject: [PATCH 076/186] Fix flakyness in RemoteStoreRefreshListenerTests (#8550) --------- Signed-off-by: Ashish Singh --- .../RemoteStoreRefreshListenerTests.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 688f29fa1f4bf..21a9393408529 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -21,9 +21,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.lease.Releasable; import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; @@ -249,10 +249,7 @@ public void testRefreshSuccessOnFirstAttempt() throws Exception { assertBusy(() -> assertEquals(0, successLatch.getCount())); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); RemoteRefreshSegmentTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); - assertEquals(0, segmentTracker.getBytesLag()); - assertEquals(0, segmentTracker.getRefreshSeqNoLag()); - assertEquals(0, segmentTracker.getTimeMsLag()); - assertEquals(0, segmentTracker.getTotalUploadsFailed()); + assertNoLagAndTotalUploadsFailed(segmentTracker, 0); } public void testRefreshSuccessOnSecondAttempt() throws Exception { @@ -273,10 +270,7 @@ public void testRefreshSuccessOnSecondAttempt() throws Exception { assertBusy(() -> assertEquals(0, successLatch.getCount())); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); RemoteRefreshSegmentTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); - assertEquals(0, segmentTracker.getBytesLag()); - assertEquals(0, segmentTracker.getRefreshSeqNoLag()); - assertEquals(0, segmentTracker.getTimeMsLag()); - assertEquals(1, segmentTracker.getTotalUploadsFailed()); + assertNoLagAndTotalUploadsFailed(segmentTracker, 1); } /** @@ -304,7 +298,7 @@ public void testRefreshSuccessAfterFailureInFirstAttemptAfterSnapshotAndMetadata assertBusy(() -> assertEquals(0, reachedCheckpointPublishLatch.getCount())); } - public void testRefreshSuccessOnThirdAttemptAttempt() throws Exception { + public void testRefreshSuccessOnThirdAttempt() throws Exception { // This covers 3 cases - 1) isRetry=false, shouldRetry=true 2) isRetry=true, shouldRetry=false 3) isRetry=True, shouldRetry=true // Succeed on 3rd attempt int succeedOnAttempt = 3; @@ -322,11 +316,16 @@ public void testRefreshSuccessOnThirdAttemptAttempt() throws Exception { assertBusy(() -> assertEquals(0, successLatch.getCount())); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); RemoteRefreshSegmentTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); - assertEquals(0, segmentTracker.getBytesLag()); - assertEquals(0, segmentTracker.getRefreshSeqNoLag()); - assertEquals(0, segmentTracker.getTimeMsLag()); - assertEquals(2, segmentTracker.getTotalUploadsFailed()); + assertNoLagAndTotalUploadsFailed(segmentTracker, 2); + } + private void assertNoLagAndTotalUploadsFailed(RemoteRefreshSegmentTracker segmentTracker, long totalUploadsFailed) throws Exception { + assertBusy(() -> { + assertEquals(0, segmentTracker.getBytesLag()); + assertEquals(0, segmentTracker.getRefreshSeqNoLag()); + assertEquals(0, segmentTracker.getTimeMsLag()); + assertEquals(totalUploadsFailed, segmentTracker.getTotalUploadsFailed()); + }); } public void testTrackerData() throws Exception { From cb0d13b9cab950b39269eb28691e6075cd9cf1aa Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Mon, 10 Jul 2023 00:35:41 -0700 Subject: [PATCH 077/186] [Optimization] Cluster State Update Optimization (#7853) * Cluster State Update Optimization - Optimize Metadata build() to skip redundant computations of indicesLookup as part of ClusterState build Signed-off-by: Sandesh Kumar --------- Signed-off-by: Sandesh Kumar --- CHANGELOG.md | 2 +- .../opensearch/cluster/metadata/Metadata.java | 44 +++++++- .../cluster/metadata/MetadataTests.java | 103 ++++++++++++++++++ 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0146af42ee3b..e7998d4022cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -157,7 +157,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) - [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) - Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) - +- Optimize Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index dde9ebfb54a49..5d0e4f9aa7e3f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -1121,12 +1121,14 @@ public static class Builder { private final Map indices; private final Map templates; private final Map customs; + private final Metadata previousMetadata; public Builder() { clusterUUID = UNKNOWN_CLUSTER_UUID; indices = new HashMap<>(); templates = new HashMap<>(); customs = new HashMap<>(); + previousMetadata = null; indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize } @@ -1141,6 +1143,7 @@ public Builder(Metadata metadata) { this.indices = new HashMap<>(metadata.indices); this.templates = new HashMap<>(metadata.templates); this.customs = new HashMap<>(metadata.customs); + this.previousMetadata = metadata; } public Builder put(IndexMetadata.Builder indexMetadataBuilder) { @@ -1425,6 +1428,44 @@ public Builder generateClusterUuidIfNeeded() { } public Metadata build() { + DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE); + DataStreamMetadata previousDataStreamMetadata = (previousMetadata != null) + ? (DataStreamMetadata) this.previousMetadata.customs.get(DataStreamMetadata.TYPE) + : null; + + boolean recomputeRequiredforIndicesLookups = (previousMetadata == null) + || (indices.equals(previousMetadata.indices) == false) + || (previousDataStreamMetadata != null && previousDataStreamMetadata.equals(dataStreamMetadata) == false) + || (dataStreamMetadata != null && dataStreamMetadata.equals(previousDataStreamMetadata) == false); + + return (recomputeRequiredforIndicesLookups == false) + ? buildMetadataWithPreviousIndicesLookups() + : buildMetadataWithRecomputedIndicesLookups(); + } + + protected Metadata buildMetadataWithPreviousIndicesLookups() { + return new Metadata( + clusterUUID, + clusterUUIDCommitted, + version, + coordinationMetadata, + transientSettings, + persistentSettings, + hashesOfConsistentSettings, + indices, + templates, + customs, + Arrays.copyOf(previousMetadata.allIndices, previousMetadata.allIndices.length), + Arrays.copyOf(previousMetadata.visibleIndices, previousMetadata.visibleIndices.length), + Arrays.copyOf(previousMetadata.allOpenIndices, previousMetadata.allOpenIndices.length), + Arrays.copyOf(previousMetadata.visibleOpenIndices, previousMetadata.visibleOpenIndices.length), + Arrays.copyOf(previousMetadata.allClosedIndices, previousMetadata.allClosedIndices.length), + Arrays.copyOf(previousMetadata.visibleClosedIndices, previousMetadata.visibleClosedIndices.length), + Collections.unmodifiableSortedMap(previousMetadata.indicesLookup) + ); + } + + protected Metadata buildMetadataWithRecomputedIndicesLookups() { // TODO: We should move these datastructures to IndexNameExpressionResolver, this will give the following benefits: // 1) The datastructures will be rebuilt only when needed. Now during serializing we rebuild these datastructures // while these datastructures aren't even used. @@ -1586,8 +1627,7 @@ private SortedMap buildIndicesLookup() { IndexAbstraction existing = indicesLookup.put(indexMetadata.getIndex().getName(), index); assert existing == null : "duplicate for " + indexMetadata.getIndex(); - for (final AliasMetadata aliasCursor : indexMetadata.getAliases().values()) { - AliasMetadata aliasMetadata = aliasCursor; + for (final AliasMetadata aliasMetadata : indexMetadata.getAliases().values()) { indicesLookup.compute(aliasMetadata.getAlias(), (aliasName, alias) -> { if (alias == null) { return new IndexAbstraction.Alias(aliasMetadata, indexMetadata); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java index a744c181ee341..29904aa9e3ff8 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java @@ -58,6 +58,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -76,6 +77,10 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class MetadataTests extends OpenSearchTestCase { @@ -1364,6 +1369,62 @@ public void testValidateDataStreamsForNullDataStreamMetadata() { } } + public void testMetadataBuildInvocations() { + final Metadata previousMetadata = randomMetadata(); + Metadata builtMetadata; + Metadata.Builder spyBuilder; + + // previous Metadata state was not provided to Builder during assignment - indices lookups should get re-computed + spyBuilder = spy(Metadata.builder()); + builtMetadata = spyBuilder.build(); + verify(spyBuilder, times(1)).buildMetadataWithRecomputedIndicesLookups(); + verify(spyBuilder, times(0)).buildMetadataWithPreviousIndicesLookups(); + compareMetadata(Metadata.EMPTY_METADATA, builtMetadata, true, true, false); + + // no changes in builder method after initialization from previous Metadata - indices lookups should not be re-computed + spyBuilder = spy(Metadata.builder(previousMetadata)); + builtMetadata = spyBuilder.version(previousMetadata.version() + 1).build(); + verify(spyBuilder, times(0)).buildMetadataWithRecomputedIndicesLookups(); + verify(spyBuilder, times(1)).buildMetadataWithPreviousIndicesLookups(); + compareMetadata(previousMetadata, builtMetadata, true, true, true); + reset(spyBuilder); + + // adding new index - all indices lookups should get re-computed + spyBuilder = spy(Metadata.builder(previousMetadata)); + String index = "new_index_" + randomAlphaOfLength(3); + builtMetadata = spyBuilder.indices( + Collections.singletonMap( + index, + IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1).build() + ) + ).build(); + verify(spyBuilder, times(1)).buildMetadataWithRecomputedIndicesLookups(); + verify(spyBuilder, times(0)).buildMetadataWithPreviousIndicesLookups(); + compareMetadata(previousMetadata, builtMetadata, false, true, false); + reset(spyBuilder); + + // adding new templates - indices lookups should not get recomputed + spyBuilder = spy(Metadata.builder(previousMetadata)); + builtMetadata = spyBuilder.put("component_template_new_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance()) + .put("index_template_v2_new_" + randomAlphaOfLength(3), ComposableIndexTemplateTests.randomInstance()) + .build(); + verify(spyBuilder, times(0)).buildMetadataWithRecomputedIndicesLookups(); + verify(spyBuilder, times(1)).buildMetadataWithPreviousIndicesLookups(); + compareMetadata(previousMetadata, builtMetadata, true, false, false); + reset(spyBuilder); + + // adding new data stream - indices lookups should get re-computed + spyBuilder = spy(Metadata.builder(previousMetadata)); + DataStream dataStream = DataStreamTests.randomInstance(); + for (Index backingIndex : dataStream.getIndices()) { + spyBuilder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(backingIndex)); + } + builtMetadata = spyBuilder.put(dataStream).version(previousMetadata.version() + 1).build(); + verify(spyBuilder, times(1)).buildMetadataWithRecomputedIndicesLookups(); + verify(spyBuilder, times(0)).buildMetadataWithPreviousIndicesLookups(); + compareMetadata(previousMetadata, builtMetadata, false, true, true); + } + public static Metadata randomMetadata() { Metadata.Builder md = Metadata.builder() .put(buildIndexMetadata("index", "alias", randomBoolean() ? null : randomBoolean()).build(), randomBoolean()) @@ -1435,4 +1496,46 @@ private static class CreateIndexResult { this.metadata = metadata; } } + + private static void compareMetadata( + final Metadata previousMetadata, + final Metadata newMetadata, + final boolean compareIndicesLookups, + final boolean compareTemplates, + final boolean checkVersionIncrement + ) { + assertEquals(previousMetadata.clusterUUID(), newMetadata.clusterUUID()); + assertEquals(previousMetadata.clusterUUIDCommitted(), newMetadata.clusterUUIDCommitted()); + assertEquals(previousMetadata.coordinationMetadata(), newMetadata.coordinationMetadata()); + assertEquals(previousMetadata.settings(), newMetadata.settings()); + assertEquals(previousMetadata.transientSettings(), newMetadata.transientSettings()); + assertEquals(previousMetadata.persistentSettings(), newMetadata.persistentSettings()); + assertEquals(previousMetadata.hashesOfConsistentSettings(), newMetadata.hashesOfConsistentSettings()); + + if (compareIndicesLookups == true) { + assertEquals(previousMetadata.indices(), newMetadata.indices()); + assertEquals(previousMetadata.getConcreteAllIndices(), newMetadata.getConcreteAllIndices()); + assertEquals(previousMetadata.getConcreteAllClosedIndices(), newMetadata.getConcreteAllClosedIndices()); + assertEquals(previousMetadata.getConcreteAllOpenIndices(), newMetadata.getConcreteAllOpenIndices()); + assertEquals(previousMetadata.getConcreteVisibleIndices(), newMetadata.getConcreteVisibleIndices()); + assertEquals(previousMetadata.getConcreteVisibleClosedIndices(), newMetadata.getConcreteVisibleClosedIndices()); + assertEquals(previousMetadata.getConcreteVisibleOpenIndices(), newMetadata.getConcreteVisibleOpenIndices()); + assertEquals(previousMetadata.getIndicesLookup(), newMetadata.getIndicesLookup()); + assertEquals(previousMetadata.getCustoms().get(DataStreamMetadata.TYPE), newMetadata.getCustoms().get(DataStreamMetadata.TYPE)); + } + + if (compareTemplates == true) { + assertEquals(previousMetadata.templates(), newMetadata.templates()); + assertEquals(previousMetadata.templatesV2(), newMetadata.templatesV2()); + assertEquals(previousMetadata.componentTemplates(), newMetadata.componentTemplates()); + } + + if (compareIndicesLookups == true && compareTemplates == true) { + assertEquals(previousMetadata.getCustoms(), newMetadata.getCustoms()); + } + + if (checkVersionIncrement == true) { + assertEquals(previousMetadata.version() + 1, newMetadata.version()); + } + } } From 418ab51718cb68b7a57d4cf524b1dec9ed02b224 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 10 Jul 2023 13:10:41 +0530 Subject: [PATCH 078/186] [Remote Segment Store] Fix flaky RemoteStoreIT.testStaleCommitDeletion tests (#8523) Signed-off-by: Sachin Kale --- .../org/opensearch/remotestore/RemoteStoreIT.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 85208a33cc9f5..f01e4969b1fe7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.hamcrest.MatcherAssert; import org.junit.Before; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; @@ -35,6 +36,8 @@ import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.comparesEqualTo; +import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.is; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -285,7 +288,6 @@ public void testRemoteTranslogCleanup() throws Exception { verifyRemoteStoreCleanup(true); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8504") public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); @@ -301,16 +303,15 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { assertBusy(() -> { int actualFileCount = getFileCount(indexPath); if (numberOfIterations <= RemoteStoreRefreshListener.LAST_N_METADATA_FILES_TO_KEEP) { - assertEquals(numberOfIterations, actualFileCount); + MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations, numberOfIterations + 1))); } else { // As delete is async its possible that the file gets created before the deletion or after // deletion. - assertTrue(actualFileCount >= 10 || actualFileCount <= 11); + MatcherAssert.assertThat(actualFileCount, is(oneOf(10, 11))); } }, 30, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8504") public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); @@ -322,6 +323,8 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { .get() .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID, "/0/segments/metadata"); - assertEquals(numberOfIterations, getFileCount(indexPath)); + int actualFileCount = getFileCount(indexPath); + // We also allow (numberOfIterations + 1) as index creation also triggers refresh. + MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations, numberOfIterations + 1))); } } From 6d39aaa4db00dcb2b33fd1c3837026f337a7c344 Mon Sep 17 00:00:00 2001 From: Ashish Date: Mon, 10 Jul 2023 19:24:44 +0530 Subject: [PATCH 079/186] Extend existing IndexRecoveryIT for remote indexes (#8505) Signed-off-by: Ashish Singh --- .../indices/recovery/IndexRecoveryIT.java | 40 ++++++--- .../remotestore/RemoteIndexRecoveryIT.java | 88 +++++++++++++++++++ .../opensearch/index/engine/NoOpEngine.java | 14 +-- .../opensearch/test/InternalTestCluster.java | 16 ++-- 4 files changed, 137 insertions(+), 21 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java index d04c31c0d6e24..72b9b32236371 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java @@ -34,6 +34,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.IndexCommit; +import org.hamcrest.Matcher; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; @@ -101,8 +102,8 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.NodeIndicesStats; import org.opensearch.indices.analysis.AnalysisModule; -import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.recovery.RecoveryState.Stage; +import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.node.NodeClosedException; import org.opensearch.node.RecoverySettingsChunkSizePlugin; import org.opensearch.plugins.AnalysisPlugin; @@ -577,21 +578,25 @@ public void testRerouteRecovery() throws Exception { .clear() .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)) .get(); - assertThat(statsResponse1.getNodes(), hasSize(2)); - for (NodeStats nodeStats : statsResponse1.getNodes()) { + List dataNodeStats = statsResponse1.getNodes() + .stream() + .filter(nodeStats -> nodeStats.getNode().isDataNode()) + .collect(Collectors.toList()); + assertThat(dataNodeStats, hasSize(2)); + for (NodeStats nodeStats : dataNodeStats) { final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats(); if (nodeStats.getNode().getName().equals(nodeA)) { assertThat( "node A throttling should increase", recoveryStats.throttleTime().millis(), - greaterThan(finalNodeAThrottling) + getMatcherForThrottling(finalNodeAThrottling) ); } if (nodeStats.getNode().getName().equals(nodeB)) { assertThat( "node B throttling should increase", recoveryStats.throttleTime().millis(), - greaterThan(finalNodeBThrottling) + getMatcherForThrottling(finalNodeBThrottling) ); } } @@ -623,7 +628,7 @@ public void testRerouteRecovery() throws Exception { final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats(); assertThat(recoveryStats.currentAsSource(), equalTo(0)); assertThat(recoveryStats.currentAsTarget(), equalTo(0)); - assertThat(nodeName + " throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L)); + assertThat(nodeName + " throttling should be >0", recoveryStats.throttleTime().millis(), getMatcherForThrottling(0)); }; // we have to use assertBusy as recovery counters are decremented only when the last reference to the RecoveryTarget // is decremented, which may happen after the recovery was done. @@ -644,7 +649,8 @@ public void testRerouteRecovery() throws Exception { logger.info("--> start node C"); String nodeC = internalCluster().startNode(); - assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut()); + int nodeCount = internalCluster().getNodeNames().length; + assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(String.valueOf(nodeCount)).get().isTimedOut()); logger.info("--> slowing down recoveries"); slowDownRecovery(shardSize); @@ -678,7 +684,7 @@ public void testRerouteRecovery() throws Exception { assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC); validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex()); - if (randomBoolean()) { + if (randomBoolean() && shouldAssertOngoingRecoveryInRerouteRecovery()) { // shutdown node with relocation source of replica shard and check if recovery continues internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA)); ensureStableCluster(2); @@ -722,6 +728,14 @@ public void testRerouteRecovery() throws Exception { validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex()); } + protected boolean shouldAssertOngoingRecoveryInRerouteRecovery() { + return false; + } + + protected Matcher getMatcherForThrottling(long value) { + return greaterThan(value); + } + public void testSnapshotRecovery() throws Exception { logger.info("--> start node A"); String nodeA = internalCluster().startNode(); @@ -824,7 +838,7 @@ private IndicesStatsResponse createAndPopulateIndex(String name, int nodeCount, ensureGreen(); logger.info("--> indexing sample data"); - final int numDocs = between(MIN_DOC_COUNT, MAX_DOC_COUNT); + final int numDocs = numDocs(); final IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { @@ -846,6 +860,10 @@ private void validateIndexRecoveryState(ReplicationLuceneIndex indexState) { assertThat(indexState.recoveredBytesPercent(), lessThanOrEqualTo(100.0f)); } + protected int numDocs() { + return between(MIN_DOC_COUNT, MAX_DOC_COUNT); + } + public void testTransientErrorsDuringRecoveryAreRetried() throws Exception { final String indexName = "test"; final Settings nodeSettings = Settings.builder() @@ -1384,10 +1402,10 @@ public void testHistoryRetention() throws Exception { flush(indexName); } - String firstNodeToStop = randomFrom(internalCluster().getNodeNames()); + String firstNodeToStop = randomFrom(internalCluster().getDataNodeNames()); Settings firstNodeToStopDataPathSettings = internalCluster().dataPathSettings(firstNodeToStop); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(firstNodeToStop)); - String secondNodeToStop = randomFrom(internalCluster().getNodeNames()); + String secondNodeToStop = randomFrom(internalCluster().getDataNodeNames()); Settings secondNodeToStopDataPathSettings = internalCluster().dataPathSettings(secondNodeToStop); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(secondNodeToStop)); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java new file mode 100644 index 0000000000000..11c9993ac7874 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.IndexModule; +import org.opensearch.index.IndexSettings; +import org.opensearch.indices.recovery.IndexRecoveryIT; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteIndexRecoveryIT extends IndexRecoveryIT { + + protected static final String REPOSITORY_NAME = "test-remore-store-repo"; + + protected Path absolutePath; + + @Override + protected Settings featureFlagSettings() { + return Settings.builder() + .put(super.featureFlagSettings()) + .put(FeatureFlags.REMOTE_STORE, "true") + .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") + .build(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + internalCluster().startClusterManagerOnlyNode(); + absolutePath = randomRepoPath().toAbsolutePath(); + assertAcked( + clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) + ); + } + + @Override + public Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + } + + @After + public void teardown() { + assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + } + + @Override + protected Matcher getMatcherForThrottling(long value) { + return Matchers.greaterThanOrEqualTo(value); + } + + @Override + protected int numDocs() { + return randomIntBetween(100, 200); + } + + @Override + protected boolean shouldAssertOngoingRecoveryInRerouteRecovery() { + return false; + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java index 2b126e627bd3d..5c548df1cbb60 100644 --- a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java @@ -209,11 +209,15 @@ public void trimUnreferencedTranslogFiles() throws TranslogException { translog.trimUnreferencedReaders(); // refresh the translog stats translogStats = translog.stats(); - assert translog.currentFileGeneration() == translog.getMinFileGeneration() : "translog was not trimmed " - + " current gen " - + translog.currentFileGeneration() - + " != min gen " - + translog.getMinFileGeneration(); + // When remote translog is enabled, the min file generation is dependent on the (N-1) + // lastRefreshedCheckpoint SeqNo - refer RemoteStoreRefreshListener. This leads to older generations not + // being trimmed and leading to current generation being higher than the min file generation. + assert engineConfig.getIndexSettings().isRemoteTranslogStoreEnabled() + || translog.currentFileGeneration() == translog.getMinFileGeneration() : "translog was not trimmed " + + " current gen " + + translog.currentFileGeneration() + + " != min gen " + + translog.getMinFileGeneration(); } } } catch (final Exception e) { diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 49d8b64bc71cd..3f7bb71b27681 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -1524,11 +1524,13 @@ public void assertSeqNos() throws Exception { } assertThat(replicaShardRouting + " seq_no_stats mismatch", seqNoStats, equalTo(primarySeqNoStats)); // the local knowledge on the primary of the global checkpoint equals the global checkpoint on the shard - assertThat( - replicaShardRouting + " global checkpoint syncs mismatch", - seqNoStats.getGlobalCheckpoint(), - equalTo(syncGlobalCheckpoints.get(replicaShardRouting.allocationId().getId())) - ); + if (primaryShard.isRemoteTranslogEnabled() == false) { + assertThat( + replicaShardRouting + " global checkpoint syncs mismatch", + seqNoStats.getGlobalCheckpoint(), + equalTo(syncGlobalCheckpoints.get(replicaShardRouting.allocationId().getId())) + ); + } } } } @@ -2155,6 +2157,10 @@ synchronized Set allDataNodesButN(int count) { return set; } + public Set getDataNodeNames() { + return allDataNodesButN(0); + } + /** * Returns a set of nodes that have at least one shard of the given index. */ From 5ad6d6df6c098a29b8521baf7909c1b31f2b8715 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 10 Jul 2023 12:03:56 -0400 Subject: [PATCH 080/186] Bump com.google.guava:guava from 30.1.1-jre to 32.1.1-jre (#8583) Signed-off-by: Andriy Redko --- CHANGELOG.md | 2 +- buildSrc/version.properties | 2 +- distribution/tools/keystore-cli/build.gradle | 2 +- distribution/tools/plugin-cli/build.gradle | 2 +- distribution/tools/upgrade-cli/build.gradle | 2 +- plugins/ingest-attachment/licenses/guava-32.0.1-jre.jar.sha1 | 1 - plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 | 1 + plugins/repository-azure/licenses/guava-32.0.1-jre.jar.sha1 | 1 - plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 | 1 + plugins/repository-gcs/licenses/guava-32.0.1-jre.jar.sha1 | 1 - plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 | 1 + plugins/repository-hdfs/licenses/guava-32.0.1-jre.jar.sha1 | 1 - plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 | 1 + 13 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 plugins/ingest-attachment/licenses/guava-32.0.1-jre.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/guava-32.0.1-jre.jar.sha1 create mode 100644 plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/guava-32.0.1-jre.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/guava-32.0.1-jre.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e7998d4022cb8..929431bba24d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,7 +105,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) -- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401) +- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401, #8581) - Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812) - Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810) - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 408b03e60cc5d..7a2ddc24aabcf 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -22,7 +22,7 @@ jettison = 1.5.4 woodstox = 6.4.0 kotlin = 1.7.10 antlr4 = 4.11.1 -guava = 32.0.1-jre +guava = 32.1.1-jre protobuf = 3.22.3 jakarta_annotation = 1.3.5 diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle index fe57b342ae298..d819322fc77b7 100644 --- a/distribution/tools/keystore-cli/build.gradle +++ b/distribution/tools/keystore-cli/build.gradle @@ -35,7 +35,7 @@ dependencies { compileOnly project(":libs:opensearch-cli") testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' - testRuntimeOnly('com.google.guava:guava:32.1.1-jre') { + testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } } diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index d39697e81914b..7a300c17c8189 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -41,7 +41,7 @@ dependencies { api "org.bouncycastle:bc-fips:1.0.2.3" testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' - testRuntimeOnly('com.google.guava:guava:32.1.1-jre') { + testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } diff --git a/distribution/tools/upgrade-cli/build.gradle b/distribution/tools/upgrade-cli/build.gradle index d81d00440a864..c7bf0ff1d2810 100644 --- a/distribution/tools/upgrade-cli/build.gradle +++ b/distribution/tools/upgrade-cli/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' - testRuntimeOnly('com.google.guava:guava:32.1.1-jre') { + testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } } diff --git a/plugins/ingest-attachment/licenses/guava-32.0.1-jre.jar.sha1 b/plugins/ingest-attachment/licenses/guava-32.0.1-jre.jar.sha1 deleted file mode 100644 index 80dc9e9308a6c..0000000000000 --- a/plugins/ingest-attachment/licenses/guava-32.0.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e5d51a72d142f2d40a57dfb897188b36a95b489 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 new file mode 100644 index 0000000000000..0d791b5d3f55b --- /dev/null +++ b/plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 @@ -0,0 +1 @@ +ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/guava-32.0.1-jre.jar.sha1 b/plugins/repository-azure/licenses/guava-32.0.1-jre.jar.sha1 deleted file mode 100644 index 80dc9e9308a6c..0000000000000 --- a/plugins/repository-azure/licenses/guava-32.0.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e5d51a72d142f2d40a57dfb897188b36a95b489 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 new file mode 100644 index 0000000000000..0d791b5d3f55b --- /dev/null +++ b/plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 @@ -0,0 +1 @@ +ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/guava-32.0.1-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-32.0.1-jre.jar.sha1 deleted file mode 100644 index 80dc9e9308a6c..0000000000000 --- a/plugins/repository-gcs/licenses/guava-32.0.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e5d51a72d142f2d40a57dfb897188b36a95b489 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 new file mode 100644 index 0000000000000..0d791b5d3f55b --- /dev/null +++ b/plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 @@ -0,0 +1 @@ +ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/guava-32.0.1-jre.jar.sha1 b/plugins/repository-hdfs/licenses/guava-32.0.1-jre.jar.sha1 deleted file mode 100644 index 80dc9e9308a6c..0000000000000 --- a/plugins/repository-hdfs/licenses/guava-32.0.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e5d51a72d142f2d40a57dfb897188b36a95b489 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 new file mode 100644 index 0000000000000..0d791b5d3f55b --- /dev/null +++ b/plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 @@ -0,0 +1 @@ +ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file From 91bfa01606974b947455fbc289e21a1aad096fa8 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Mon, 10 Jul 2023 09:43:19 -0700 Subject: [PATCH 081/186] Add safeguard limits for file cache during node level allocation (#8208) Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 1 + .../cluster/ClusterInfoServiceIT.java | 28 +++ .../org/opensearch/cluster/ClusterInfo.java | 24 +- .../cluster/InternalClusterInfoService.java | 15 +- .../decider/DiskThresholdDecider.java | 52 +++++ .../store/remote/filecache/FileCache.java | 3 + .../opensearch/cluster/ClusterInfoTests.java | 24 +- .../allocation/DiskThresholdMonitorTests.java | 2 +- ...dexShardConstraintDeciderOverlapTests.java | 2 +- .../RemoteShardsBalancerBaseTestCase.java | 2 +- .../decider/DiskThresholdDeciderTests.java | 205 +++++++++++++++++- .../DiskThresholdDeciderUnitTests.java | 13 +- .../MockInternalClusterInfoService.java | 3 +- .../opensearch/test/OpenSearchTestCase.java | 8 + 14 files changed, 367 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929431bba24d5..ecb92a4051738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,6 +158,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) - Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) - Optimize Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853)) +- Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterInfoServiceIT.java index 17e8526acfd74..508b8e21e42c1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterInfoServiceIT.java @@ -50,6 +50,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndexDescriptor; @@ -192,6 +193,11 @@ public void testClusterInfoServiceCollectsInformation() { logger.info("--> shard size: {}", size); assertThat("shard size is greater than 0", size, greaterThanOrEqualTo(0L)); } + + final Map nodeFileCacheStats = info.nodeFileCacheStats; + assertNotNull(nodeFileCacheStats); + assertThat("file cache is empty on non search nodes", nodeFileCacheStats.size(), Matchers.equalTo(0)); + ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getClusterManagerName()); ClusterState state = clusterService.state(); for (ShardRouting shard : state.routingTable().allShards()) { @@ -209,6 +215,28 @@ public void testClusterInfoServiceCollectsInformation() { } } + public void testClusterInfoServiceCollectsFileCacheInformation() { + internalCluster().startNodes(1); + internalCluster().ensureAtLeastNumSearchAndDataNodes(2); + + InternalTestCluster internalTestCluster = internalCluster(); + // Get the cluster info service on the cluster-manager node + final InternalClusterInfoService infoService = (InternalClusterInfoService) internalTestCluster.getInstance( + ClusterInfoService.class, + internalTestCluster.getClusterManagerName() + ); + infoService.setUpdateFrequency(TimeValue.timeValueMillis(200)); + ClusterInfo info = infoService.refresh(); + assertNotNull("info should not be null", info); + final Map nodeFileCacheStats = info.nodeFileCacheStats; + assertNotNull(nodeFileCacheStats); + assertThat("file cache is enabled on both search nodes", nodeFileCacheStats.size(), Matchers.equalTo(2)); + + for (FileCacheStats fileCacheStats : nodeFileCacheStats.values()) { + assertThat("file cache is non empty", fileCacheStats.getTotal().getBytes(), greaterThan(0L)); + } + } + public void testClusterInfoServiceInformationClearOnError() { internalCluster().startNodes( 2, diff --git a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java index 876a36c205975..ffa3d0d19fb71 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java @@ -34,6 +34,7 @@ import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.opensearch.Version; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -42,6 +43,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.shard.ShardId; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import java.io.IOException; import java.util.Collections; @@ -63,9 +65,10 @@ public class ClusterInfo implements ToXContentFragment, Writeable { public static final ClusterInfo EMPTY = new ClusterInfo(); final Map routingToDataPath; final Map reservedSpace; + final Map nodeFileCacheStats; protected ClusterInfo() { - this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); + this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); } /** @@ -83,13 +86,15 @@ public ClusterInfo( final Map mostAvailableSpaceUsage, final Map shardSizes, final Map routingToDataPath, - final Map reservedSpace + final Map reservedSpace, + final Map nodeFileCacheStats ) { this.leastAvailableSpaceUsage = leastAvailableSpaceUsage; this.shardSizes = shardSizes; this.mostAvailableSpaceUsage = mostAvailableSpaceUsage; this.routingToDataPath = routingToDataPath; this.reservedSpace = reservedSpace; + this.nodeFileCacheStats = nodeFileCacheStats; } public ClusterInfo(StreamInput in) throws IOException { @@ -105,6 +110,11 @@ public ClusterInfo(StreamInput in) throws IOException { this.shardSizes = Collections.unmodifiableMap(sizeMap); this.routingToDataPath = Collections.unmodifiableMap(routingMap); this.reservedSpace = Collections.unmodifiableMap(reservedSpaceMap); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.nodeFileCacheStats = in.readMap(StreamInput::readString, FileCacheStats::new); + } else { + this.nodeFileCacheStats = Map.of(); + } } @Override @@ -114,6 +124,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.shardSizes, StreamOutput::writeString, (o, v) -> out.writeLong(v == null ? -1 : v)); out.writeMap(this.routingToDataPath, (o, k) -> k.writeTo(o), StreamOutput::writeString); out.writeMap(this.reservedSpace, (o, v) -> v.writeTo(o), (o, v) -> v.writeTo(o)); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeMap(this.nodeFileCacheStats, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + } } public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { @@ -187,6 +200,13 @@ public Map getNodeMostAvailableDiskUsages() { return Collections.unmodifiableMap(this.mostAvailableSpaceUsage); } + /** + * Returns a node id to file cache stats mapping for the nodes that have search roles assigned to it. + */ + public Map getNodeFileCacheStats() { + return Collections.unmodifiableMap(this.nodeFileCacheStats); + } + /** * Returns the shard size for the given shard routing or null it that metric is not available. */ diff --git a/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java index 0acc7bece439f..9c12d6bb3e7ea 100644 --- a/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java @@ -59,6 +59,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.store.StoreStats; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ReceiveTimeoutTransportException; @@ -72,6 +73,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.stream.Collectors; /** * InternalClusterInfoService provides the ClusterInfoService interface, @@ -110,6 +112,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private volatile Map leastAvailableSpaceUsages; private volatile Map mostAvailableSpaceUsages; + private volatile Map nodeFileCacheStats; private volatile IndicesStatsSummary indicesStatsSummary; // null if this node is not currently the cluster-manager private final AtomicReference refreshAndRescheduleRunnable = new AtomicReference<>(); @@ -122,6 +125,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); + this.nodeFileCacheStats = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; @@ -208,7 +212,8 @@ public ClusterInfo getClusterInfo() { mostAvailableSpaceUsages, indicesStatsSummary.shardSizes, indicesStatsSummary.shardRoutingToDataPath, - indicesStatsSummary.reservedSpace + indicesStatsSummary.reservedSpace, + nodeFileCacheStats ); } @@ -221,6 +226,7 @@ protected CountDownLatch updateNodeStats(final ActionListener(listener, latch)); return latch; @@ -264,6 +270,13 @@ public void onResponse(NodesStatsResponse nodesStatsResponse) { ); leastAvailableSpaceUsages = Collections.unmodifiableMap(leastAvailableUsagesBuilder); mostAvailableSpaceUsages = Collections.unmodifiableMap(mostAvailableUsagesBuilder); + + nodeFileCacheStats = Collections.unmodifiableMap( + nodesStatsResponse.getNodes() + .stream() + .filter(nodeStats -> nodeStats.getNode().isSearchNode()) + .collect(Collectors.toMap(nodeStats -> nodeStats.getNode().getId(), NodeStats::getFileCacheStats)) + ); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index ddd5e9274f08b..e216ca4511bff 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -54,14 +54,21 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.snapshots.SnapshotShardSizeInfo; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import static org.opensearch.cluster.routing.RoutingPool.REMOTE_CAPABLE; +import static org.opensearch.cluster.routing.RoutingPool.getNodePool; +import static org.opensearch.cluster.routing.RoutingPool.getShardPool; import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING; import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; +import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO; /** * The {@link DiskThresholdDecider} checks that the node a shard is potentially @@ -167,6 +174,42 @@ public static long sizeOfRelocatingShards( @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { ClusterInfo clusterInfo = allocation.clusterInfo(); + + /* + The following block enables allocation for remote shards within safeguard limits of the filecache. + */ + if (REMOTE_CAPABLE.equals(getNodePool(node)) && REMOTE_CAPABLE.equals(getShardPool(shardRouting, allocation))) { + final List remoteShardsOnNode = StreamSupport.stream(node.spliterator(), false) + .filter(shard -> shard.primary() && REMOTE_CAPABLE.equals(getShardPool(shard, allocation))) + .collect(Collectors.toList()); + final long currentNodeRemoteShardSize = remoteShardsOnNode.stream() + .map(ShardRouting::getExpectedShardSize) + .mapToLong(Long::longValue) + .sum(); + + final long shardSize = getExpectedShardSize( + shardRouting, + 0L, + allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), + allocation.metadata(), + allocation.routingTable() + ); + + final FileCacheStats fileCacheStats = clusterInfo.getNodeFileCacheStats().getOrDefault(node.nodeId(), null); + final long nodeCacheSize = fileCacheStats != null ? fileCacheStats.getTotal().getBytes() : 0; + final long totalNodeRemoteShardSize = currentNodeRemoteShardSize + shardSize; + + if (totalNodeRemoteShardSize > DATA_TO_FILE_CACHE_SIZE_RATIO * nodeCacheSize) { + return allocation.decision( + Decision.NO, + NAME, + "file cache limit reached - remote shard size will exceed configured safeguard ratio" + ); + } + return Decision.YES; + } + Map usages = clusterInfo.getNodeMostAvailableDiskUsages(); final Decision decision = earlyTerminate(allocation, usages); if (decision != null) { @@ -422,6 +465,15 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl if (shardRouting.currentNodeId().equals(node.nodeId()) == false) { throw new IllegalArgumentException("Shard [" + shardRouting + "] is not allocated on node: [" + node.nodeId() + "]"); } + + /* + The following block prevents movement for remote shards since they do not use the local storage as + the primary source of data storage. + */ + if (REMOTE_CAPABLE.equals(getNodePool(node)) && REMOTE_CAPABLE.equals(getShardPool(shardRouting, allocation))) { + return Decision.ALWAYS; + } + final ClusterInfo clusterInfo = allocation.clusterInfo(); final Map usages = clusterInfo.getNodeLeastAvailableDiskUsages(); final Decision decision = earlyTerminate(allocation, usages); diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java index 0aa3740fb6ecb..3d23b4d22538c 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java @@ -49,6 +49,9 @@ public class FileCache implements RefCountedCache { private final CircuitBreaker circuitBreaker; + // TODO: Convert the constant into an integer setting + public static final int DATA_TO_FILE_CACHE_SIZE_RATIO = 5; + public FileCache(SegmentedCache cache, CircuitBreaker circuitBreaker) { this.theCache = cache; this.circuitBreaker = circuitBreaker; diff --git a/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java index a32d6e35d0182..e1294da1e57bc 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java @@ -36,6 +36,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.index.shard.ShardId; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.test.OpenSearchTestCase; import java.util.HashMap; @@ -49,7 +50,8 @@ public void testSerialization() throws Exception { randomDiskUsage(), randomShardSizes(), randomRoutingToDataPath(), - randomReservedSpace() + randomReservedSpace(), + randomFileCacheStats() ); BytesStreamOutput output = new BytesStreamOutput(); clusterInfo.writeTo(output); @@ -60,6 +62,7 @@ public void testSerialization() throws Exception { assertEquals(clusterInfo.shardSizes, result.shardSizes); assertEquals(clusterInfo.routingToDataPath, result.routingToDataPath); assertEquals(clusterInfo.reservedSpace, result.reservedSpace); + assertEquals(clusterInfo.getNodeFileCacheStats().size(), result.getNodeFileCacheStats().size()); } private static Map randomDiskUsage() { @@ -79,6 +82,25 @@ private static Map randomDiskUsage() { return builder; } + private static Map randomFileCacheStats() { + int numEntries = randomIntBetween(0, 16); + final Map builder = new HashMap<>(numEntries); + for (int i = 0; i < numEntries; i++) { + String key = randomAlphaOfLength(16); + FileCacheStats fileCacheStats = new FileCacheStats( + randomLong(), + randomLong(), + randomLong(), + randomLong(), + randomLong(), + randomLong(), + randomLong() + ); + builder.put(key, fileCacheStats); + } + return builder; + } + private static Map randomShardSizes() { int numEntries = randomIntBetween(0, 128); final Map builder = new HashMap<>(numEntries); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 21d891bdbc317..3e21f6c19e150 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -798,7 +798,7 @@ private static ClusterInfo clusterInfo( final Map diskUsages, final Map reservedSpace ) { - return new ClusterInfo(diskUsages, null, null, null, reservedSpace); + return new ClusterInfo(diskUsages, null, null, null, reservedSpace, Map.of()); } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java index 7112af6b4efc0..15dcae65ce6e7 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java @@ -176,7 +176,7 @@ public DevNullClusterInfo( final Map shardSizes, final Map reservedSpace ) { - super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, reservedSpace); + super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, reservedSpace, Map.of()); } @Override diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsBalancerBaseTestCase.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsBalancerBaseTestCase.java index 9d7d0ebc5b2b1..dbb08a999877d 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsBalancerBaseTestCase.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsBalancerBaseTestCase.java @@ -239,7 +239,7 @@ public DevNullClusterInfo( final Map mostAvailableSpaceUsage, final Map shardSizes ) { - super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, Map.of()); + super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, Map.of(), Map.of()); } @Override diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index c23d98c95fc3c..4ccf0a9bc3a20 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -70,6 +70,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.EmptySnapshotsInfoService; import org.opensearch.snapshots.InternalSnapshotsInfoService.SnapshotShard; @@ -83,6 +84,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; @@ -283,6 +285,190 @@ public void testDiskThreshold() { assertThat(clusterState.getRoutingNodes().node("node4").size(), equalTo(1)); } + public void testDiskThresholdForRemoteShards() { + Settings diskSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8) + .build(); + + Map usages = new HashMap<>(); + usages.put("node1", new DiskUsage("node1", "node1", "/dev/null", 100, 10)); // 90% used + usages.put("node2", new DiskUsage("node2", "node2", "/dev/null", 100, 35)); // 65% used + usages.put("node3", new DiskUsage("node3", "node3", "/dev/null", 100, 60)); // 40% used + + Map shardSizes = new HashMap<>(); + shardSizes.put("[test][0][p]", 10L); // 10 bytes + shardSizes.put("[test][0][r]", 10L); + + Map fileCacheStatsMap = new HashMap<>(); + fileCacheStatsMap.put("node1", new FileCacheStats(0, 0, 1000, 0, 0, 0, 0)); + fileCacheStatsMap.put("node2", new FileCacheStats(0, 0, 1000, 0, 0, 0, 0)); + fileCacheStatsMap.put("node3", new FileCacheStats(0, 0, 1000, 0, 0, 0, 0)); + final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes, fileCacheStatsMap); + + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + AllocationDeciders deciders = new AllocationDeciders( + new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings))) + ); + + ClusterInfoService cis = () -> { + logger.info("--> calling fake getClusterInfo"); + return clusterInfo; + }; + AllocationService strategy = new AllocationService( + deciders, + new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), + cis, + EmptySnapshotsInfoService.INSTANCE + ); + + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(remoteIndexSettings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .build(); + + final RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(initialRoutingTable) + .build(); + + Set defaultWithSearchRole = new HashSet<>(CLUSTER_MANAGER_DATA_ROLES); + defaultWithSearchRole.add(DiscoveryNodeRole.SEARCH_ROLE); + + logger.info("--> adding two nodes"); + clusterState = ClusterState.builder(clusterState) + .nodes(DiscoveryNodes.builder().add(newNode("node1", defaultWithSearchRole)).add(newNode("node2", defaultWithSearchRole))) + .build(); + clusterState = strategy.reroute(clusterState, "reroute"); + logShardStates(clusterState); + + // Primary shard should be initializing, replica should not + assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2)); + + logger.info("--> start the shards (primaries)"); + clusterState = startInitializingShardsAndReroute(strategy, clusterState); + + logShardStates(clusterState); + // Assert that we're able to start the primary + assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> adding node3"); + + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build(); + clusterState = strategy.reroute(clusterState, "reroute"); + + logShardStates(clusterState); + // Assert that the replica is initialized now that node3 is available with enough space + assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(0)); + + logger.info("--> start the shards (replicas)"); + clusterState = startInitializingShardsAndReroute(strategy, clusterState); + + logShardStates(clusterState); + // Assert that the replica couldn't be started since node1 doesn't have enough space + assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0)); + } + + public void testFileCacheRemoteShardsDecisions() { + Settings diskSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "60%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "70%") + .build(); + + // We have an index with 2 primary shards each taking 40 bytes. Each node has 100 bytes available + final Map usages = new HashMap<>(); + usages.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 20)); // 80% used + usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 100)); // 0% used + + final Map shardSizes = new HashMap<>(); + shardSizes.put("[test][0][p]", 40L); + shardSizes.put("[test][1][p]", 40L); + shardSizes.put("[foo][0][p]", 10L); + + // First node has filecache size as 0, second has 1000, greater than the shard sizes. + Map fileCacheStatsMap = new HashMap<>(); + fileCacheStatsMap.put("node1", new FileCacheStats(0, 0, 0, 0, 0, 0, 0)); + fileCacheStatsMap.put("node2", new FileCacheStats(0, 0, 1000, 0, 0, 0, 0)); + + final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes, fileCacheStatsMap); + + Set defaultWithSearchRole = new HashSet<>(CLUSTER_MANAGER_DATA_ROLES); + defaultWithSearchRole.add(DiscoveryNodeRole.SEARCH_ROLE); + + DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings); + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(remoteIndexSettings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .build(); + + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + + DiscoveryNode discoveryNode1 = new DiscoveryNode( + "node1", + buildNewFakeTransportAddress(), + emptyMap(), + defaultWithSearchRole, + Version.CURRENT + ); + DiscoveryNode discoveryNode2 = new DiscoveryNode( + "node2", + buildNewFakeTransportAddress(), + emptyMap(), + defaultWithSearchRole, + Version.CURRENT + ); + DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(discoveryNode1).add(discoveryNode2).build(); + + ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(initialRoutingTable) + .nodes(discoveryNodes) + .build(); + + // Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here + ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED); + ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, true, ShardRoutingState.STARTED); + RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting); + RoutingNode secondRoutingNode = new RoutingNode("node2", discoveryNode2); + + RoutingTable.Builder builder = RoutingTable.builder() + .add( + IndexRoutingTable.builder(firstRouting.index()) + .addIndexShard(new IndexShardRoutingTable.Builder(firstRouting.shardId()).addShard(firstRouting).build()) + .addIndexShard(new IndexShardRoutingTable.Builder(secondRouting.shardId()).addShard(secondRouting).build()) + ); + ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); + RoutingAllocation routingAllocation = new RoutingAllocation( + null, + new RoutingNodes(clusterState), + clusterState, + clusterInfo, + null, + System.nanoTime() + ); + routingAllocation.debugDecision(true); + Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); + assertThat(decision.type(), equalTo(Decision.Type.YES)); + + decision = diskThresholdDecider.canAllocate(firstRouting, firstRoutingNode, routingAllocation); + assertThat(decision.type(), equalTo(Decision.Type.NO)); + + assertThat( + ((Decision.Single) decision).getExplanation(), + containsString("file cache limit reached - remote shard size will exceed configured safeguard ratio") + ); + + decision = diskThresholdDecider.canAllocate(firstRouting, secondRoutingNode, routingAllocation); + assertThat(decision.type(), equalTo(Decision.Type.YES)); + } + public void testDiskThresholdWithAbsoluteSizes() { Settings diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) @@ -863,7 +1049,8 @@ public void testShardRelocationsTakenIntoAccount() { Map.of( new ClusterInfo.NodeAndPath("node1", "/dev/null"), new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), between(51, 200)).build() - ) + ), + Map.of() ) ); clusterState = applyStartedShardsUntilNoChange(clusterState, strategy); @@ -1455,16 +1642,26 @@ static class DevNullClusterInfo extends ClusterInfo { final Map mostAvailableSpaceUsage, final Map shardSizes ) { - this(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of()); + this(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), Map.of()); + } + + DevNullClusterInfo( + final Map leastAvailableSpaceUsage, + final Map mostAvailableSpaceUsage, + final Map shardSizes, + final Map nodeFileCacheStats + ) { + this(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), nodeFileCacheStats); } DevNullClusterInfo( final Map leastAvailableSpaceUsage, final Map mostAvailableSpaceUsage, final Map shardSizes, - Map reservedSpace + Map reservedSpace, + final Map nodeFileCacheStats ) { - super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, reservedSpace); + super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, reservedSpace, nodeFileCacheStats); } @Override diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index caab381e65e84..62c52e93aad33 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -127,7 +127,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { final Map shardSizes = new HashMap<>(); shardSizes.put("[test][0][p]", 10L); // 10 bytes - final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of()); + final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of(), Map.of()); RoutingAllocation allocation = new RoutingAllocation( new AllocationDeciders(Collections.singleton(decider)), clusterState.getRoutingNodes(), @@ -203,7 +203,7 @@ public void testCannotAllocateDueToLackOfDiskResources() { // way bigger than available space final long shardSize = randomIntBetween(110, 1000); shardSizes.put("[test][0][p]", shardSize); - ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of()); + ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of(), Map.of()); RoutingAllocation allocation = new RoutingAllocation( new AllocationDeciders(Collections.singleton(decider)), clusterState.getRoutingNodes(), @@ -320,7 +320,14 @@ public void testCanRemainUsesLeastAvailableSpace() { shardSizes.put("[test][1][p]", 10L); shardSizes.put("[test][2][p]", 10L); - final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, shardRoutingMap, Map.of()); + final ClusterInfo clusterInfo = new ClusterInfo( + leastAvailableUsages, + mostAvailableUsage, + shardSizes, + shardRoutingMap, + Map.of(), + Map.of() + ); RoutingAllocation allocation = new RoutingAllocation( new AllocationDeciders(Collections.singleton(decider)), clusterState.getRoutingNodes(), diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java index 6634d1b4dbafc..6354cf18e8b62 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java @@ -132,7 +132,8 @@ class SizeFakingClusterInfo extends ClusterInfo { delegate.getNodeMostAvailableDiskUsages(), delegate.shardSizes, delegate.routingToDataPath, - delegate.reservedSpace + delegate.reservedSpace, + delegate.nodeFileCacheStats ); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 7722b59313b5f..ec397a2baa640 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -109,6 +109,7 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; import org.opensearch.index.Index; +import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.analysis.AnalyzerScope; @@ -1200,6 +1201,13 @@ public static Settings.Builder settings(Version version) { return builder; } + public static Settings.Builder remoteIndexSettings(Version version) { + Settings.Builder builder = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, version) + .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.REMOTE_SNAPSHOT.getSettingsKey()); + return builder; + } + /** * Returns size random values */ From 2b8b846f02c308ed82253acb8cf3ea62f652246a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 12:44:43 -0400 Subject: [PATCH 082/186] Bump com.netflix.nebula:gradle-info-plugin from 12.1.4 to 12.1.5 (#8568) * Bump com.netflix.nebula:gradle-info-plugin from 12.1.4 to 12.1.5 Bumps [com.netflix.nebula:gradle-info-plugin](https://github.com/nebula-plugins/gradle-info-plugin) from 12.1.4 to 12.1.5. - [Release notes](https://github.com/nebula-plugins/gradle-info-plugin/releases) - [Changelog](https://github.com/nebula-plugins/gradle-info-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/nebula-plugins/gradle-info-plugin/compare/v12.1.4...v12.1.5) --- updated-dependencies: - dependency-name: com.netflix.nebula:gradle-info-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 4 ++-- buildSrc/build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecb92a4051738..74a66a7880114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,7 +125,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016)) - Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122)) - Bump `com.networknt:json-schema-validator` from 1.0.83 to 1.0.84 (#8141) -- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.4 (#8139) +- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.5 (#8139, #8568) - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140)) - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181)) - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) @@ -177,4 +177,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index feb8da7c20984..852f9ef7f0474 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -114,7 +114,7 @@ dependencies { api 'org.apache.ant:ant:1.10.13' api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0' api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0' - api 'com.netflix.nebula:gradle-info-plugin:12.1.4' + api 'com.netflix.nebula:gradle-info-plugin:12.1.5' api 'org.apache.rat:apache-rat:0.15' api 'commons-io:commons-io:2.13.0' api "net.java.dev.jna:jna:5.13.0" From 8b44a5dc624f2a12ad256b290e612e55069e9470 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 10 Jul 2023 12:50:14 -0400 Subject: [PATCH 083/186] Manually update google cloud core from 2.17.0 to 2.21.0 (#8586) * Update google cloud core Signed-off-by: Stephen Crawford * Specify pr number Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford --- CHANGELOG.md | 8 ++++---- plugins/repository-gcs/build.gradle | 2 +- .../licenses/google-cloud-core-http-2.17.0.jar.sha1 | 1 - .../licenses/google-cloud-core-http-2.21.0.jar.sha1 | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-2.17.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a66a7880114..77d8099225b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,10 +37,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344) - Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928))) - Add Opentelemetry dependencies ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) -- Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) -- Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) -- Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - +- Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) ### Changed diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 41c36dffea296..9bf1e8ac856b1 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -67,7 +67,7 @@ dependencies { api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" api 'com.google.cloud:google-cloud-core:2.5.10' - api 'com.google.cloud:google-cloud-core-http:2.17.0' + api 'com.google.cloud:google-cloud-core-http:2.21.0' api 'com.google.cloud:google-cloud-storage:1.113.1' api 'com.google.code.gson:gson:2.9.0' diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.17.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.17.0.jar.sha1 deleted file mode 100644 index eaf69a96b62b9..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.17.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b9a2fc2235dadfa359967a3d67e8bb11eb62a6dd \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 new file mode 100644 index 0000000000000..2ef0a9bf9b33e --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 @@ -0,0 +1 @@ +07da4710ccdbcfee253672c0b9e00e7370626c26 \ No newline at end of file From a15f0ed56d2787439963ac88fe836c11c720d66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:07:59 -0700 Subject: [PATCH 084/186] Bump com.azure:azure-storage-blob from 12.22.2 to 12.22.3 in /plugins/repository-azure (#8572) * Bump com.azure:azure-storage-blob in /plugins/repository-azure Bumps [com.azure:azure-storage-blob](https://github.com/Azure/azure-sdk-for-java) from 12.22.2 to 12.22.3. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-blob_12.22.2...azure-storage-blob_12.22.3) --- updated-dependencies: - dependency-name: com.azure:azure-storage-blob dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-blob-12.22.2.jar.sha1 | 1 - .../licenses/azure-storage-blob-12.22.3.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.22.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.22.3.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 77d8099225b73..fb58a54b6c9ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434)) - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) +- Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 4edb9e0b1913e..9ec1b4ee50569 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -55,7 +55,7 @@ dependencies { api "io.netty:netty-resolver-dns:${versions.netty}" api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') - api 'com.azure:azure-storage-blob:12.22.2' + api 'com.azure:azure-storage-blob:12.22.3' api 'org.reactivestreams:reactive-streams:1.0.4' api 'io.projectreactor:reactor-core:3.5.6' api 'io.projectreactor.netty:reactor-netty:1.1.8' diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.22.2.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.22.2.jar.sha1 deleted file mode 100644 index a03bb750a0a96..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-blob-12.22.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1441a678a0d28ed3b22efc27fef4752f91502834 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.22.3.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.22.3.jar.sha1 new file mode 100644 index 0000000000000..f6c3cc6e579fa --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-blob-12.22.3.jar.sha1 @@ -0,0 +1 @@ +0df12462c2eac3beaf25d283f707a0560853228b \ No newline at end of file From 828730f23377e78dd1df0fa53939ed7f2486800e Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Tue, 11 Jul 2023 02:22:46 +0900 Subject: [PATCH 085/186] replace with getField (#7855) * replace with getField Signed-off-by: Shinsuke Sugaya * Update server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java Co-authored-by: Andriy Redko Signed-off-by: Shinsuke Sugaya * add import Signed-off-by: Shinsuke Sugaya * add PR to CHANGELOG Signed-off-by: Shinsuke Sugaya * move SortedSetDocValuesField addition for field type Signed-off-by: Shinsuke Sugaya * remove a test case for flat-object field Signed-off-by: Shinsuke Sugaya * remove getField Signed-off-by: Mingshi Liu * remove unused import Signed-off-by: Shinsuke Sugaya --------- Signed-off-by: Shinsuke Sugaya Signed-off-by: Shinsuke Sugaya Signed-off-by: Mingshi Liu Co-authored-by: Andriy Redko Co-authored-by: Mingshi Liu --- CHANGELOG.md | 1 + .../rest-api-spec/test/painless/30_search.yml | 97 ------------------- .../index/mapper/FlatObjectFieldMapper.java | 23 ++--- 3 files changed, 11 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb58a54b6c9ab..7dc484744c606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,6 +174,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591)) - Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318)) - Precise system clock time in MasterService debug logs ([#7902](https://github.com/opensearch-project/OpenSearch/pull/7902)) +- Improve indexing performance for flat_object type ([#7855](https://github.com/opensearch-project/OpenSearch/pull/7855)) ### Security diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml index 4b3d5bd9e2980..a006fde630716 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml @@ -482,100 +482,3 @@ }] - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.root_cause.0.reason: "script score function must not produce negative scores, but got: [-9.0]"} - ---- - -"Flat-object fields from within the scripting": - - skip: - version: " - 2.6.99" - reason: "flat_object is introduced in 2.7.0" - - - do: - indices.create: - index: test - body: - mappings: - properties: - flat: - type : "flat_object" - - # This document has 6 distinct parts in its flat_object field paths: - # - flat.field_1 - # - flat.field_2 - # - flat.field_3 - # - flat.inner - # - flat.field_A - # - flat.field_B - - do: - index: - index: test - id: 1 - body: { - "flat": { - "field_1": "John Doe", - "field_2": 33, - "field_3": false, - "inner": { - "field_A": ["foo", "bar"], - "field_B": false - } - } - } - - - do: - index: - index: test - id: 2 - body: { - "flat": { - "field_1": "Joe Public", - "field_2": 45 - } - } - - - do: - indices.refresh: - index: test - - # It is possible to filter based on the number of distinct parts of flat_object field paths - - do: - search: - body: { - _source: true, - query: { - bool: { - filter: { - script: { - script: { - source: "doc['flat'].size() == 6", - lang: "painless" - } - } - } - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.flat.field_1: "John Doe" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - filter: { - script: { - script: { - source: "doc['flat'].size() < 6", - lang: "painless" - } - } - } - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.flat.field_1: "Joe Public" } diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 36e0adbbf057f..f8206d138534d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -659,21 +659,18 @@ private void parseValueAddFields(ParseContext context, String value, String fiel } if (fieldType().hasDocValues()) { - if (context.doc().getField(fieldType().name()) == null || !context.doc().getFields(fieldType().name()).equals(field)) { - if (fieldName.equals(fieldType().name())) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } - if (valueType.equals(VALUE_SUFFIX)) { - if (valueFieldMapper != null) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); - } + if (fieldName.equals(fieldType().name())) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + } + if (valueType.equals(VALUE_SUFFIX)) { + if (valueFieldMapper != null) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); } - if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - if (valueAndPathFieldMapper != null) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); - } + } + if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + if (valueAndPathFieldMapper != null) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); } - } } From 0bb6eb809f6ea50b620849670c6575cb3219285e Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 10 Jul 2023 13:56:51 -0400 Subject: [PATCH 086/186] Update Gradle to 8.2.1 (#8580) Signed-off-by: Andriy Redko --- buildSrc/build.gradle | 25 ++++++++++++------------ gradle/wrapper/gradle-wrapper.properties | 4 ++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 852f9ef7f0474..b976085389da3 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -34,6 +34,7 @@ import org.gradle.util.GradleVersion plugins { id 'java-gradle-plugin' id 'groovy' + id 'java-test-fixtures' } group = 'org.opensearch.gradle' @@ -78,17 +79,9 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) { } sourceSets { - test { - java { - srcDirs += ['src/testFixtures/java'] - } - } integTest { compileClasspath += sourceSets["main"].output + configurations["testRuntimeClasspath"] runtimeClasspath += output + compileClasspath - java { - srcDirs += ['src/testFixtures/java'] - } } } @@ -131,10 +124,10 @@ dependencies { api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" api "org.ajoberstar.grgit:grgit-core:5.2.0" - testImplementation "junit:junit:${props.getProperty('junit')}" - testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" - testRuntimeOnly gradleApi() - testRuntimeOnly gradleTestKit() + testFixturesApi "junit:junit:${props.getProperty('junit')}" + testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" + testFixturesApi gradleApi() + testFixturesApi gradleTestKit() testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.35.0' testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}" integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') { @@ -183,7 +176,7 @@ if (project != rootProject) { // build-tools is not ready for primetime with these... tasks.named("dependencyLicenses").configure { it.enabled = false } dependenciesInfo.enabled = false - disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest') + disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures') jarHell.enabled = false thirdPartyAudit.enabled = false if (org.opensearch.gradle.info.BuildParams.inFipsJvm) { @@ -250,6 +243,12 @@ if (project != rootProject) { } } + // disable fail-on-warnings for this specific task which trips Java 11 bug + // https://bugs.openjdk.java.net/browse/JDK-8209058 + tasks.named("compileTestFixturesJava").configure { + options.compilerArgs -= '-Werror' + } + tasks.register("integTest", Test) { inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE) systemProperty 'test.version_under_test', version diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f00d0c8442459..e10ceefe2a012 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640 +distributionSha256Sum=7c3ad722e9b0ce8205b91560fd6ce8296ac3eadf065672242fd73c06b8eeb6ee From e3341452dfadfeb0a03832f285c7090b10d61fa4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 14:02:42 -0400 Subject: [PATCH 087/186] Bump com.google.jimfs:jimfs from 1.2 to 1.3.0 in /distribution/tools/keystore-cli (#8577) * Bump com.google.jimfs:jimfs in /distribution/tools/keystore-cli Bumps [com.google.jimfs:jimfs](https://github.com/google/jimfs) from 1.2 to 1.3.0. - [Release notes](https://github.com/google/jimfs/releases) - [Commits](https://github.com/google/jimfs/compare/v1.2...v1.3.0) --- updated-dependencies: - dependency-name: com.google.jimfs:jimfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + distribution/tools/keystore-cli/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dc484744c606..f112a7ed008ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) - Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) +- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle index d819322fc77b7..5dcddf3ef127e 100644 --- a/distribution/tools/keystore-cli/build.gradle +++ b/distribution/tools/keystore-cli/build.gradle @@ -34,7 +34,7 @@ dependencies { compileOnly project(":server") compileOnly project(":libs:opensearch-cli") testImplementation project(":test:framework") - testImplementation 'com.google.jimfs:jimfs:1.2' + testImplementation 'com.google.jimfs:jimfs:1.3.0' testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } From 71f807329a92f05012924348de0b5d8884023bd0 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 10 Jul 2023 14:21:40 -0400 Subject: [PATCH 088/186] Bump schema validator from from 1.0.85 to 1.0.86 (#8573) * Bump schema validator Signed-off-by: Stephen Crawford * Update changelog Signed-off-by: Stephen Crawford * Manually trigger retry Signed-off-by: Stephen Crawford * Make trivial change to trigger rerun Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> --- CHANGELOG.md | 1 + buildSrc/build.gradle | 2 +- .../java/org/opensearch/identity/IdentityService.java | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f112a7ed008ee..b86e83e3181ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) - Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b976085389da3..018b63816c3f1 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -118,7 +118,7 @@ dependencies { api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.9.3' - api 'com.networknt:json-schema-validator:1.0.85' + api 'com.networknt:json-schema-validator:1.0.86' api 'org.jruby.jcodings:jcodings:1.0.58' api 'org.jruby.joni:joni:2.2.1' api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" diff --git a/server/src/main/java/org/opensearch/identity/IdentityService.java b/server/src/main/java/org/opensearch/identity/IdentityService.java index ab1456cd860ac..1ce107f743efc 100644 --- a/server/src/main/java/org/opensearch/identity/IdentityService.java +++ b/server/src/main/java/org/opensearch/identity/IdentityService.java @@ -5,15 +5,15 @@ package org.opensearch.identity; +import java.util.List; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; -import org.opensearch.identity.noop.NoopIdentityPlugin; -import java.util.List; import org.opensearch.common.settings.Settings; +import org.opensearch.identity.noop.NoopIdentityPlugin; import org.opensearch.identity.tokens.TokenManager; import org.opensearch.plugins.IdentityPlugin; -import java.util.stream.Collectors; /** * Identity and access control for OpenSearch. @@ -44,7 +44,7 @@ public IdentityService(final Settings settings, final List ident } /** - * Gets the current subject + * Gets the current Subject */ public Subject getSubject() { return identityPlugin.getSubject(); From bcadd172f0df7d93bbc669ff3070880538f5ecfb Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 10 Jul 2023 16:03:54 -0400 Subject: [PATCH 089/186] Move changelog entry (#8599) Signed-off-by: Stephen Crawford --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b86e83e3181ea..d303a2b4b50c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) -- Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) - Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) @@ -140,6 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) - Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) - Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577) +- Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) @@ -181,4 +181,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x From 62b66e56b9eddda1313a71ca90f2e95f45cdec5d Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Mon, 10 Jul 2023 15:35:23 -0500 Subject: [PATCH 090/186] add jdk.incubator.vector module support for JDK 20+ (#8601) * add jdk.incubator.vector module support for JDK 20+ Adds support for the incubating jdk vector package (PANAMA) when using jdk 20+ runtime. Signed-off-by: Nicholas Walter Knize * update changelog and fix typo Signed-off-by: Nicholas Walter Knize --------- Signed-off-by: Nicholas Walter Knize --- CHANGELOG.md | 1 + build.gradle | 3 +++ distribution/src/config/jvm.options | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d303a2b4b50c6..b43885a65a4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) - Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) - Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) +- Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/build.gradle b/build.gradle index ca4c6c3635d57..6a14ab231894b 100644 --- a/build.gradle +++ b/build.gradle @@ -424,6 +424,9 @@ gradle.projectsEvaluated { if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) { task.jvmArgs += ["-Djava.security.manager=allow"] } + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) { + task.jvmArgs += ["--add-modules=jdk.incubator.vector"] + } } } diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index ef1035489c9fc..e15afc0f677c3 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -78,3 +78,7 @@ ${error.file} # Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380) 18-:-Djava.security.manager=allow + +# JDK 20+ Incubating Vector Module for SIMD optimizations; +# disabling may reduce performance on vector optimized lucene +20:--add-modules=jdk.incubator.vector From 397069f578ff99efbdc0b2eb0d97c7cd2b8dba52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 14:10:13 -0700 Subject: [PATCH 091/186] Bump com.google.jimfs:jimfs in /distribution/tools/upgrade-cli (#8571) Bumps [com.google.jimfs:jimfs](https://github.com/google/jimfs) from 1.2 to 1.3.0. - [Release notes](https://github.com/google/jimfs/releases) - [Commits](https://github.com/google/jimfs/compare/v1.2...v1.3.0) --- updated-dependencies: - dependency-name: com.google.jimfs:jimfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: owaiskazi19 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- distribution/tools/upgrade-cli/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b43885a65a4d6..dee395cbc8962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,7 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) - Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) -- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577) +- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) - Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) ### Changed diff --git a/distribution/tools/upgrade-cli/build.gradle b/distribution/tools/upgrade-cli/build.gradle index c7bf0ff1d2810..99824463f14f8 100644 --- a/distribution/tools/upgrade-cli/build.gradle +++ b/distribution/tools/upgrade-cli/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" testImplementation project(":test:framework") - testImplementation 'com.google.jimfs:jimfs:1.2' + testImplementation 'com.google.jimfs:jimfs:1.3.0' testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } From a39f60f482c405a102e6eec83097e91acc90a889 Mon Sep 17 00:00:00 2001 From: Shivansh Arora <31575408+shiv0408@users.noreply.github.com> Date: Tue, 11 Jul 2023 04:54:33 +0530 Subject: [PATCH 092/186] Fix painless casting bug causing opensearch to crash (#8315) * Created a failing test to reproduce painless bug Signed-off-by: Shivansh Arora * Removed unused import Signed-off-by: Shivansh Arora * Throw exception when trying to cast def to void Signed-off-by: Shivansh Arora * Removed update context change Signed-off-by: Shivansh Arora * Created a different test Signed-off-by: Shivansh Arora --------- Signed-off-by: Shivansh Arora --- CHANGELOG.md | 1 + .../main/java/org/opensearch/painless/AnalyzerCaster.java | 2 +- .../test/java/org/opensearch/painless/FactoryTests.java | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dee395cbc8962..724a75e4e0405 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230)) - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541)) - Adds log4j configuration for telemetry LogSpanExporter ([#8393](https://github.com/opensearch-project/OpenSearch/pull/8393)) +- Fix painless casting bug, which crashes the OpenSearch process ([#8315](https://github.com/opensearch-project/OpenSearch/pull/8315)) ### Security diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/opensearch/painless/AnalyzerCaster.java index e375ff14db67e..d830ef2ab6290 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/AnalyzerCaster.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/AnalyzerCaster.java @@ -412,7 +412,7 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas } } - if (actual == def.class + if ((actual == def.class && expected != void.class) || (actual != void.class && expected == def.class) || expected.isAssignableFrom(actual) || (actual.isAssignableFrom(expected) && explicit)) { diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/FactoryTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/FactoryTests.java index 95b18cf1c5250..b4e322e12bc45 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/FactoryTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/FactoryTests.java @@ -360,6 +360,14 @@ public void testVoidReturn() { assertEquals(iae.getMessage(), "not a statement: result not used from addition operation [+]"); } + public void testDefToVoidReturnThrowsException() { + ClassCastException exception = expectScriptThrows( + ClassCastException.class, + () -> getEngine().compile("def_return_in_void", "def x=1;return x;", VoidReturnTestScript.CONTEXT, Collections.emptyMap()) + ); + assertEquals(exception.getMessage(), "Cannot cast from [def] to [void]."); + } + public abstract static class FactoryTestConverterScript { private final Map params; From 3c1bca4e28db20b525c010b5d03af850cd899217 Mon Sep 17 00:00:00 2001 From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:14:16 +0530 Subject: [PATCH 093/186] Disabled translog fsync in remote store path (#8288) Signed-off-by: Vikas Bansal --- .../opensearch/index/translog/Checkpoint.java | 10 ++- .../index/translog/TranslogHeader.java | 6 +- .../index/translog/TranslogWriter.java | 29 ++++-- .../translog/TruncateTranslogAction.java | 2 +- .../index/translog/LocalTranslogTests.java | 85 ++++++++++++++++++ .../index/translog/RemoteFSTranslogTests.java | 90 +++++++++++++++++++ .../index/translog/TranslogHeaderTests.java | 4 +- 7 files changed, 208 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/Checkpoint.java b/server/src/main/java/org/opensearch/index/translog/Checkpoint.java index 8df574ed8374f..56de7e5daf55f 100644 --- a/server/src/main/java/org/opensearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/opensearch/index/translog/Checkpoint.java @@ -223,12 +223,14 @@ public static void write(ChannelFactory factory, Path checkpointFile, Checkpoint } } - public static void write(FileChannel fileChannel, Path checkpointFile, Checkpoint checkpoint) throws IOException { + public static void write(FileChannel fileChannel, Path checkpointFile, Checkpoint checkpoint, boolean fsync) throws IOException { byte[] bytes = createCheckpointBytes(checkpointFile, checkpoint); Channels.writeToChannel(bytes, fileChannel, 0); - // no need to force metadata, file size stays the same and we did the full fsync - // when we first created the file, so the directory entry doesn't change as well - fileChannel.force(false); + if (fsync == true) { + // no need to force metadata, file size stays the same and we did the full fsync + // when we first created the file, so the directory entry doesn't change as well + fileChannel.force(false); + } } private static byte[] createCheckpointBytes(Path checkpointFile, Checkpoint checkpoint) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index 0819d009c9992..af6ebcf7b7c66 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -213,7 +213,7 @@ private static void tryReportOldVersionError(final Path path, final FileChannel /** * Writes this header with the latest format into the file channel */ - void write(final FileChannel channel) throws IOException { + void write(final FileChannel channel, boolean fsync) throws IOException { // This output is intentionally not closed because closing it will close the FileChannel. @SuppressWarnings({ "IOResourceOpenedButNotSafelyClosed", "resource" }) final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput( @@ -229,7 +229,9 @@ void write(final FileChannel channel) throws IOException { // Checksum header out.writeInt((int) out.getChecksum()); out.flush(); - channel.force(true); + if (fsync == true) { + channel.force(true); + } assert channel.position() == headerSizeInBytes : "Header is not fully written; header size [" + headerSizeInBytes + "], channel position [" diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 423e9dd960ed7..e19aece60adc0 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -110,6 +110,8 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable { private final Map> seenSequenceNumbers; + private final Boolean remoteTranslogEnabled; + private TranslogWriter( final ShardId shardId, final Checkpoint initialCheckpoint, @@ -123,7 +125,8 @@ private TranslogWriter( TranslogHeader header, final TragicExceptionHolder tragedy, final LongConsumer persistedSequenceNumberConsumer, - final BigArrays bigArrays + final BigArrays bigArrays, + Boolean remoteTranslogEnabled ) throws IOException { super(initialCheckpoint.generation, channel, path, header); assert initialCheckpoint.offset == channel.position() : "initial checkpoint offset [" @@ -148,6 +151,7 @@ private TranslogWriter( this.bigArrays = bigArrays; this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null; this.tragedy = tragedy; + this.remoteTranslogEnabled = remoteTranslogEnabled; } public static TranslogWriter create( @@ -174,14 +178,14 @@ public static TranslogWriter create( try { checkpointChannel = channelFactory.open(checkpointFile, StandardOpenOption.WRITE); final TranslogHeader header = new TranslogHeader(translogUUID, primaryTerm); - header.write(channel); + header.write(channel, !Boolean.TRUE.equals(remoteTranslogEnabled)); final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint( header.sizeInBytes(), fileGeneration, initialGlobalCheckpoint, initialMinTranslogGen ); - writeCheckpoint(checkpointChannel, checkpointFile, checkpoint); + writeCheckpoint(checkpointChannel, checkpointFile, checkpoint, remoteTranslogEnabled); final LongSupplier writerGlobalCheckpointSupplier; if (Assertions.ENABLED) { writerGlobalCheckpointSupplier = () -> { @@ -209,7 +213,8 @@ public static TranslogWriter create( header, tragedy, persistedSequenceNumberConsumer, - bigArrays + bigArrays, + remoteTranslogEnabled ); } catch (Exception exception) { // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that @@ -508,8 +513,10 @@ final boolean syncUpTo(long offset) throws IOException { // now do the actual fsync outside of the synchronized block such that // we can continue writing to the buffer etc. try { - channel.force(false); - writeCheckpoint(checkpointChannel, checkpointPath, checkpointToSync); + if (!Boolean.TRUE.equals(remoteTranslogEnabled)) { + channel.force(false); + } + writeCheckpoint(checkpointChannel, checkpointPath, checkpointToSync, remoteTranslogEnabled); } catch (final Exception ex) { closeWithTragicEvent(ex); throw ex; @@ -603,9 +610,13 @@ protected void readBytes(ByteBuffer targetBuffer, long position) throws IOExcept Channels.readFromFileChannelWithEofException(channel, position, targetBuffer); } - private static void writeCheckpoint(final FileChannel fileChannel, final Path checkpointFile, final Checkpoint checkpoint) - throws IOException { - Checkpoint.write(fileChannel, checkpointFile, checkpoint); + private static void writeCheckpoint( + final FileChannel fileChannel, + final Path checkpointFile, + final Checkpoint checkpoint, + final Boolean remoteTranslogEnabled + ) throws IOException { + Checkpoint.write(fileChannel, checkpointFile, checkpoint, !Boolean.TRUE.equals(remoteTranslogEnabled)); } /** diff --git a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java index ef948cfade815..4a082b4a19844 100644 --- a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java +++ b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java @@ -258,7 +258,7 @@ private static void writeEmptyCheckpoint(Path filename, int translogLength, long private static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException { try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) { TranslogHeader header = new TranslogHeader(translogUUID, SequenceNumbers.UNASSIGNED_PRIMARY_TERM); - header.write(fc); + header.write(fc, true); return header.sizeInBytes(); } } diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java index b2827b8bc2953..a7d7b3a51cebb 100644 --- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java @@ -1541,6 +1541,91 @@ ChannelFactory getChannelFactory() { } } + public void testTranslogWriterFsyncedWithLocalTranslog() throws IOException { + Path tempDir = createTempDir(); + final TranslogConfig temp = getTranslogConfig(tempDir); + final TranslogConfig config = new TranslogConfig( + temp.getShardId(), + temp.getTranslogPath(), + temp.getIndexSettings(), + temp.getBigArrays(), + new ByteSizeValue(1, ByteSizeUnit.KB) + ); + + final Set persistedSeqNos = new HashSet<>(); + final AtomicInteger translogFsyncCalls = new AtomicInteger(); + final AtomicInteger checkpointFsyncCalls = new AtomicInteger(); + + final ChannelFactory channelFactory = (file, openOption) -> { + FileChannel delegate = FileChannel.open(file, openOption); + boolean success = false; + try { + // don't do partial writes for checkpoints we rely on the fact that the bytes are written as an atomic operation + final boolean isCkpFile = file.getFileName().toString().endsWith(".ckp"); + + final FileChannel channel; + if (isCkpFile) { + channel = new FilterFileChannel(delegate) { + @Override + public void force(boolean metaData) throws IOException { + checkpointFsyncCalls.incrementAndGet(); + } + }; + } else { + channel = new FilterFileChannel(delegate) { + + @Override + public void force(boolean metaData) throws IOException { + translogFsyncCalls.incrementAndGet(); + } + }; + } + success = true; + return channel; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(delegate); + } + } + }; + + String translogUUID = Translog.createEmptyTranslog( + config.getTranslogPath(), + SequenceNumbers.NO_OPS_PERFORMED, + shardId, + channelFactory, + primaryTerm.get() + ); + + try ( + Translog translog = new LocalTranslog( + config, + translogUUID, + new DefaultTranslogDeletionPolicy(-1, -1, 0), + () -> SequenceNumbers.NO_OPS_PERFORMED, + primaryTerm::get, + persistedSeqNos::add + ) { + @Override + ChannelFactory getChannelFactory() { + return channelFactory; + } + } + ) { + TranslogWriter writer = translog.getCurrent(); + byte[] bytes = new byte[256]; + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 2); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 3); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 4); + writer.sync(); + assertEquals(4, checkpointFsyncCalls.get()); + assertEquals(3, translogFsyncCalls.get()); + // Sequence numbers are marked as persisted after sync + assertThat(persistedSeqNos, contains(1L, 2L, 3L, 4L)); + } + } + public void testTranslogWriterDoesNotBlockAddsOnWrite() throws IOException, InterruptedException { Path tempDir = createTempDir(); final TranslogConfig config = getTranslogConfig(tempDir); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index 30c04c731d1f8..04dc90cd9087d 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -182,6 +182,7 @@ private TranslogConfig getTranslogConfig(final Path path) { // only randomize between nog age retention and a long one, so failures will have a chance of reproducing .put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomBoolean() ? "-1ms" : "1h") .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomIntBetween(-1, 2048) + "b") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .build(); return getTranslogConfig(path, settings); @@ -1261,6 +1262,95 @@ ChannelFactory getChannelFactory() { } } + public void testTranslogWriterFsyncDisabledInRemoteFsTranslog() throws IOException { + Path tempDir = createTempDir(); + final TranslogConfig temp = getTranslogConfig(tempDir); + final TranslogConfig config = new TranslogConfig( + temp.getShardId(), + temp.getTranslogPath(), + temp.getIndexSettings(), + temp.getBigArrays(), + new ByteSizeValue(1, ByteSizeUnit.KB) + ); + + final Set persistedSeqNos = new HashSet<>(); + final AtomicInteger translogFsyncCalls = new AtomicInteger(); + final AtomicInteger checkpointFsyncCalls = new AtomicInteger(); + + final ChannelFactory channelFactory = (file, openOption) -> { + FileChannel delegate = FileChannel.open(file, openOption); + boolean success = false; + try { + // don't do partial writes for checkpoints we rely on the fact that the bytes are written as an atomic operation + final boolean isCkpFile = file.getFileName().toString().endsWith(".ckp"); + + final FileChannel channel; + if (isCkpFile) { + channel = new FilterFileChannel(delegate) { + @Override + public void force(boolean metaData) throws IOException { + checkpointFsyncCalls.incrementAndGet(); + } + }; + } else { + channel = new FilterFileChannel(delegate) { + + @Override + public void force(boolean metaData) throws IOException { + translogFsyncCalls.incrementAndGet(); + } + }; + } + success = true; + return channel; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(delegate); + } + } + }; + + String translogUUID = Translog.createEmptyTranslog( + config.getTranslogPath(), + SequenceNumbers.NO_OPS_PERFORMED, + shardId, + channelFactory, + primaryTerm.get() + ); + + try ( + Translog translog = new RemoteFsTranslog( + config, + translogUUID, + new DefaultTranslogDeletionPolicy(-1, -1, 0), + () -> SequenceNumbers.NO_OPS_PERFORMED, + primaryTerm::get, + persistedSeqNos::add, + repository, + threadPool, + () -> Boolean.TRUE + ) { + @Override + ChannelFactory getChannelFactory() { + return channelFactory; + } + } + ) { + TranslogWriter writer = translog.getCurrent(); + byte[] bytes = new byte[256]; + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 2); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 3); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 4); + writer.sync(); + // Fsync is still enabled during empty translog creation. + assertEquals(2, checkpointFsyncCalls.get()); + assertEquals(1, translogFsyncCalls.get()); + // Sequence numbers are marked as persisted after sync + assertThat(persistedSeqNos, contains(1L, 2L, 3L, 4L)); + } + } + public void testCloseIntoReader() throws IOException { try (TranslogWriter writer = translog.createWriter(translog.currentFileGeneration() + 1)) { final int numOps = randomIntBetween(8, 128); diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java index 3569d596a5569..10b20ab207927 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java @@ -63,7 +63,7 @@ public void testCurrentHeaderVersion() throws Exception { final long generation = randomNonNegativeLong(); final Path translogFile = createTempDir().resolve(Translog.getFilename(generation)); try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { - outHeader.write(channel); + outHeader.write(channel, true); assertThat(outHeader.sizeInBytes(), equalTo((int) channel.position())); } try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { @@ -165,7 +165,7 @@ public void testCorruptTranslogHeader() throws Exception { final Path translogLocation = createTempDir(); final Path translogFile = translogLocation.resolve(Translog.getFilename(generation)); try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { - outHeader.write(channel); + outHeader.write(channel, true); assertThat(outHeader.sizeInBytes(), equalTo((int) channel.position())); } TestTranslog.corruptFile(logger, random(), translogFile, false); From a5752cb28d8705800b1118c89ecfda660b9543ce Mon Sep 17 00:00:00 2001 From: Bansi Kasundra <66969140+kasundra07@users.noreply.github.com> Date: Mon, 10 Jul 2023 21:59:05 -0700 Subject: [PATCH 094/186] =?UTF-8?q?[Snapshot=20Interop]=20Add=20Changes=20?= =?UTF-8?q?in=20Snapshot=20Status=20Flow=20for=20remote=20sto=E2=80=A6=20(?= =?UTF-8?q?#7495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bansi Kasundra --- .../blobstore/url/URLBlobContainer.java | 16 +- .../opensearch/snapshots/CloneSnapshotIT.java | 23 +-- .../snapshots/SnapshotStatusApisIT.java | 171 ++++++++++++++++++ .../BlobStoreIndexShardSnapshot.java | 16 +- .../blobstore/IndexShardSnapshot.java | 21 +++ .../RemoteStoreShardShallowCopySnapshot.java | 16 +- .../RemoteSnapshotDirectoryFactory.java | 6 +- .../blobstore/BlobStoreRepository.java | 63 +++---- .../AbstractSnapshotIntegTestCase.java | 7 + 9 files changed, 283 insertions(+), 56 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/snapshots/blobstore/IndexShardSnapshot.java diff --git a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java index 1f966be98bdf7..b13a4d5a39a5b 100644 --- a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java +++ b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java @@ -83,12 +83,20 @@ public URL url() { } /** - * This operation is not supported by URLBlobContainer + * Tests whether a blob with the given blob name exists in the container. + * + * @param blobName + * The name of the blob whose existence is to be determined. + * @return {@code true} if a blob exists in the {@link BlobContainer} with the given name, and {@code false} otherwise. */ @Override - public boolean blobExists(String blobName) { - assert false : "should never be called for a read-only url repo"; - throw new UnsupportedOperationException("URL repository doesn't support this operation"); + public boolean blobExists(String blobName) throws IOException { + try { + readBlob(blobName); + return true; + } catch (FileNotFoundException e) { + return false; + } } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index ce92a15026b70..3de982f89ac80 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -43,6 +43,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.snapshots.blobstore.IndexShardSnapshot; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryData; import org.opensearch.snapshots.mockstore.MockRepository; @@ -832,18 +833,14 @@ private static BlobStoreIndexShardSnapshot readShardSnapshot( RepositoryShardId repositoryShardId, SnapshotId snapshotId ) { - return PlainActionFuture.get( - f -> repository.threadPool() - .generic() - .execute( - ActionRunnable.supply( - f, - () -> repository.loadShardSnapshot( - repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), - snapshotId - ) - ) - ) - ); + return PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> { + IndexShardSnapshot indexShardSnapshot = repository.loadShardSnapshot( + repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), + snapshotId + ); + assert indexShardSnapshot instanceof BlobStoreIndexShardSnapshot + : "indexShardSnapshot should be an instance of BlobStoreIndexShardSnapshot"; + return (BlobStoreIndexShardSnapshot) indexShardSnapshot; + }))); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index 72c64b56c19c1..c22dd90cc930b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -48,6 +48,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.Strings; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -76,6 +77,7 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that check by-timestamp order + .put(FeatureFlags.REMOTE_STORE, "true") .build(); } @@ -110,6 +112,61 @@ public void testStatusApiConsistency() { assertEquals(snStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime()); } + public void testStatusAPICallForShallowCopySnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndex(indexName); + ensureGreen(); + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final String snapshot = "snapshot"; + createFullSnapshot(snapshotRepoName, snapshot); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + + final SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, snapshot); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + + final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName); + assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); + assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L)); + assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0)); + assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L)); + + // Validating that the incremental file count and incremental file size is zero for shallow copy + final SnapshotIndexShardStatus shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); + assertThat(shallowSnapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + assertThat(shallowSnapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); + assertThat(shallowSnapshotShardState.getStats().getTotalSize(), greaterThan(0L)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), is(0)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), is(0L)); + } + public void testStatusAPICallInProgressSnapshot() throws Exception { createRepository("test-repo", "mock", Settings.builder().put("location", randomRepoPath()).put("block_on_data", true)); @@ -188,6 +245,63 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { ); } + public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndex(indexName); + ensureGreen(); + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + createFullSnapshot(snapshotRepoName, "test-snap-1"); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + + SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, "test-snap-1"); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + + SnapshotIndexShardStatus shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); + assertThat(shallowSnapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + final int totalFileCount = shallowSnapshotShardState.getStats().getTotalFileCount(); + final long totalSize = shallowSnapshotShardState.getStats().getTotalSize(); + final int incrementalFileCount = shallowSnapshotShardState.getStats().getIncrementalFileCount(); + final long incrementalSize = shallowSnapshotShardState.getStats().getIncrementalSize(); + + createFullSnapshot(snapshotRepoName, "test-snap-2"); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); + + snapshotStatus = getSnapshotStatus(snapshotRepoName, "test-snap-2"); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); + assertThat(shallowSnapshotShardState.getStats().getTotalFileCount(), equalTo(totalFileCount)); + assertThat(shallowSnapshotShardState.getStats().getTotalSize(), equalTo(totalSize)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), equalTo(incrementalFileCount)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), equalTo(incrementalSize)); + } + public void testGetSnapshotsWithoutIndices() throws Exception { createRepository("test-repo", "fs"); @@ -326,6 +440,63 @@ public void testSnapshotStatusOnFailedSnapshot() throws Exception { assertEquals(SnapshotsInProgress.State.FAILED, snapshotsStatusResponse.getSnapshots().get(0).getState()); } + public void testStatusAPICallInProgressShallowSnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy().put("block_on_data", true)); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndex(indexName); + ensureGreen(); + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + logger.info("--> snapshot"); + ActionFuture createSnapshotResponseActionFuture = startFullSnapshot(snapshotRepoName, "test-snap"); + + logger.info("--> wait for data nodes to get blocked"); + waitForBlockOnAnyDataNode(snapshotRepoName, TimeValue.timeValueMinutes(1)); + awaitNumberOfSnapshotsInProgress(1); + assertEquals( + SnapshotsInProgress.State.STARTED, + client().admin() + .cluster() + .prepareSnapshotStatus(snapshotRepoName) + .setSnapshots("test-snap") + .get() + .getSnapshots() + .get(0) + .getState() + ); + + logger.info("--> unblock all data nodes"); + unblockAllDataNodes(snapshotRepoName); + + logger.info("--> wait for snapshot to finish"); + createSnapshotResponseActionFuture.actionGet(); + } + public void testGetSnapshotsRequest() throws Exception { final String repositoryName = "test-repo"; final String indexName = "test-idx"; diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 9d7dfa92a48eb..d295027a857cd 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -44,6 +44,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.store.StoreFileMetadata; import java.io.IOException; @@ -57,7 +58,7 @@ * * @opensearch.internal */ -public class BlobStoreIndexShardSnapshot implements ToXContentFragment { +public class BlobStoreIndexShardSnapshot implements ToXContentFragment, IndexShardSnapshot { /** * Information about snapshotted file @@ -592,4 +593,17 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th incrementalSize ); } + + @Override + public IndexShardSnapshotStatus getIndexShardSnapshotStatus() { + return IndexShardSnapshotStatus.newDone( + startTime, + time, + incrementalFileCount, + totalFileCount(), + incrementalSize, + totalSize(), + null + ); // Not adding a real generation here as it doesn't matter to callers + } } diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/IndexShardSnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/IndexShardSnapshot.java new file mode 100644 index 0000000000000..e79b9069ef16a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/IndexShardSnapshot.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.snapshots.blobstore; + +import org.opensearch.index.snapshots.IndexShardSnapshotStatus; + +/** + * Base interface for shard snapshot status + * + * @opensearch.internal + */ +@FunctionalInterface +public interface IndexShardSnapshot { + IndexShardSnapshotStatus getIndexShardSnapshotStatus(); +} diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java index 8cb9fd3cd3c63..eefc1469a06a0 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java @@ -13,6 +13,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import java.io.IOException; import java.util.ArrayList; @@ -23,7 +24,7 @@ * * @opensearch.internal */ -public class RemoteStoreShardShallowCopySnapshot implements ToXContentFragment { +public class RemoteStoreShardShallowCopySnapshot implements ToXContentFragment, IndexShardSnapshot { private final String snapshot; private final String version; @@ -433,4 +434,17 @@ public RemoteStoreShardShallowCopySnapshot asClone(String targetSnapshotName, lo fileNames ); } + + @Override + public IndexShardSnapshotStatus getIndexShardSnapshotStatus() { + return IndexShardSnapshotStatus.newDone( + startTime, + time, + incrementalFileCount(), + totalFileCount, + incrementalSize(), + totalSize, + null + ); // Not adding a real generation here as it doesn't matter to callers + } } diff --git a/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java index 2c7e66b9a121d..3238ffe45e0a6 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java @@ -22,6 +22,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; +import org.opensearch.index.snapshots.blobstore.IndexShardSnapshot; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.utils.TransferManager; import org.opensearch.plugins.IndexStorePlugin; @@ -89,7 +90,10 @@ private Future createRemoteSnapshotDirectoryFromSnapsho // index restore is invoked return threadPool.executor(ThreadPool.Names.SNAPSHOT).submit(() -> { final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(blobPath); - final BlobStoreIndexShardSnapshot snapshot = blobStoreRepository.loadShardSnapshot(blobContainer, snapshotId); + final IndexShardSnapshot indexShardSnapshot = blobStoreRepository.loadShardSnapshot(blobContainer, snapshotId); + assert indexShardSnapshot instanceof BlobStoreIndexShardSnapshot + : "indexShardSnapshot should be an instance of BlobStoreIndexShardSnapshot"; + final BlobStoreIndexShardSnapshot snapshot = (BlobStoreIndexShardSnapshot) indexShardSnapshot; TransferManager transferManager = new TransferManager(blobContainer, remoteStoreFileCache); return new RemoteSnapshotDirectory(snapshot, localStoreDir, transferManager); }); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index be5fbf2ab6a51..3e77a7e796375 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -109,6 +109,7 @@ import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots; +import org.opensearch.index.snapshots.blobstore.IndexShardSnapshot; import org.opensearch.index.snapshots.blobstore.RateLimitingInputStream; import org.opensearch.index.snapshots.blobstore.SlicedInputStream; import org.opensearch.index.snapshots.blobstore.SnapshotFiles; @@ -566,7 +567,10 @@ public void cloneShardSnapshot( // We don't need to check if there exists a shallow snapshot with the same name as we have the check before starting the clone // operation ensuring that the snapshot name is available by checking the repository data. Also, the new clone snapshot would // have a different UUID and hence a new unique snap-N file will be created. - final BlobStoreIndexShardSnapshot sourceMeta = loadShardSnapshot(shardContainer, source); + IndexShardSnapshot indexShardSnapshot = loadShardSnapshot(shardContainer, source); + assert indexShardSnapshot instanceof BlobStoreIndexShardSnapshot + : "indexShardSnapshot should be an instance of BlobStoreIndexShardSnapshot"; + final BlobStoreIndexShardSnapshot sourceMeta = (BlobStoreIndexShardSnapshot) indexShardSnapshot; logger.trace("[{}] [{}] writing shard snapshot file for clone", shardId, target); INDEX_SHARD_SNAPSHOT_FORMAT.write( sourceMeta.asClone(target.getName(), startTime, threadPool.absoluteTimeInMillis() - startTime), @@ -606,7 +610,10 @@ public void cloneRemoteStoreIndexShardSnapshot( // We don't need to check if there exists a shallow/full copy snapshot with the same name as we have the check before starting // the clone operation ensuring that the snapshot name is available by checking the repository data. Also, the new clone shallow // snapshot would have a different UUID and hence a new unique shallow-snap-N file will be created. - RemoteStoreShardShallowCopySnapshot remStoreBasedShardMetadata = loadShallowCopyShardSnapshot(shardContainer, source); + IndexShardSnapshot indexShardSnapshot = loadShardSnapshot(shardContainer, source); + assert indexShardSnapshot instanceof RemoteStoreShardShallowCopySnapshot + : "indexShardSnapshot should be an instance of RemoteStoreShardShallowCopySnapshot"; + RemoteStoreShardShallowCopySnapshot remStoreBasedShardMetadata = (RemoteStoreShardShallowCopySnapshot) indexShardSnapshot; String indexUUID = remStoreBasedShardMetadata.getIndexUUID(); String remoteStoreRepository = remStoreBasedShardMetadata.getRemoteStoreRepository(); RemoteStoreMetadataLockManager remoteStoreMetadataLockManger = remoteStoreLockManagerFactory.newLockManager( @@ -2697,7 +2704,10 @@ public void restoreShard( final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); final BlobContainer container = shardContainer(indexId, snapshotShardId); executor.execute(ActionRunnable.wrap(restoreListener, l -> { - final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId); + IndexShardSnapshot indexShardSnapshot = loadShardSnapshot(container, snapshotId); + assert indexShardSnapshot instanceof BlobStoreIndexShardSnapshot + : "indexShardSnapshot should be an instance of BlobStoreIndexShardSnapshot"; + final BlobStoreIndexShardSnapshot snapshot = (BlobStoreIndexShardSnapshot) indexShardSnapshot; final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), null); new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) { @Override @@ -2846,21 +2856,16 @@ public RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetadat ShardId snapshotShardId ) { final BlobContainer container = shardContainer(indexId, snapshotShardId); - return loadShallowCopyShardSnapshot(container, snapshotId); + IndexShardSnapshot indexShardSnapshot = loadShardSnapshot(container, snapshotId); + assert indexShardSnapshot instanceof RemoteStoreShardShallowCopySnapshot + : "indexShardSnapshot should be an instance of RemoteStoreShardShallowCopySnapshot"; + return (RemoteStoreShardShallowCopySnapshot) indexShardSnapshot; } @Override public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { - BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(shardContainer(indexId, shardId), snapshotId); - return IndexShardSnapshotStatus.newDone( - snapshot.startTime(), - snapshot.time(), - snapshot.incrementalFileCount(), - snapshot.totalFileCount(), - snapshot.incrementalSize(), - snapshot.totalSize(), - null - ); // Not adding a real generation here as it doesn't matter to callers + IndexShardSnapshot snapshot = loadShardSnapshot(shardContainer(indexId, shardId), snapshotId); + return snapshot.getIndexShardSnapshotStatus(); } @Override @@ -3019,32 +3024,18 @@ private static List unusedBlobs( .collect(Collectors.toList()); } - /** - * Loads information about remote store enabled shard snapshot for remote store interop enabled snapshots - */ - public RemoteStoreShardShallowCopySnapshot loadShallowCopyShardSnapshot(BlobContainer shardContainer, SnapshotId snapshotId) { - try { - return REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read(shardContainer, snapshotId.getUUID(), namedXContentRegistry); - } catch (NoSuchFileException ex) { - throw new SnapshotMissingException(metadata.name(), snapshotId, ex); - } catch (IOException ex) { - throw new SnapshotException( - metadata.name(), - snapshotId, - "failed to read shard snapshot file for [" + shardContainer.path() + ']', - ex - ); - } - } - /** * Loads information about shard snapshot */ - public BlobStoreIndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, SnapshotId snapshotId) { + public IndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, SnapshotId snapshotId) { try { - return INDEX_SHARD_SNAPSHOT_FORMAT.read(shardContainer, snapshotId.getUUID(), namedXContentRegistry); - } catch (NoSuchFileException ex) { - throw new SnapshotMissingException(metadata.name(), snapshotId, ex); + if (shardContainer.blobExists(INDEX_SHARD_SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()))) { + return INDEX_SHARD_SNAPSHOT_FORMAT.read(shardContainer, snapshotId.getUUID(), namedXContentRegistry); + } else if (shardContainer.blobExists(REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()))) { + return REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read(shardContainer, snapshotId.getUUID(), namedXContentRegistry); + } else { + throw new SnapshotMissingException(metadata.name(), snapshotId.getName()); + } } catch (IOException ex) { throw new SnapshotException( metadata.name(), diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index 83051c7fed4e4..de5a569bebaf9 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -425,6 +425,13 @@ protected Settings.Builder randomRepositorySettings() { return settings; } + protected Settings.Builder snapshotRepoSettingsForShallowCopy() { + final Settings.Builder settings = Settings.builder(); + settings.put("location", randomRepoPath()); + settings.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE); + return settings; + } + protected static Settings.Builder indexSettingsNoReplicas(int shards) { return Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); } From 11f46994a6a0e4fa510fb878891446ccb007fbf4 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Tue, 11 Jul 2023 02:34:00 -0400 Subject: [PATCH 095/186] Manually bump net.minidev:json-smart from 2.4.11 to 2.5.0 (#8576) * Bump schema validator Signed-off-by: Stephen Crawford * update Json smart Signed-off-by: Stephen Crawford * Update changelog Signed-off-by: Stephen Crawford * Manually retry Signed-off-by: Stephen Crawford * Remove unneeded comment to trigger rerun Signed-off-by: Stephen Crawford * MOve line Signed-off-by: Stephen Crawford * Update changelog Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> --- CHANGELOG.md | 4 ++-- test/fixtures/hdfs-fixture/build.gradle | 2 +- .../src/main/java/hdfs/MiniHDFS.java | 24 +++++++++---------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 724a75e4e0405..5130dd00845b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) -- Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) - ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -140,8 +138,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) - Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) +- Bump `net.minidev:json-smart` from 2.4.11 to 2.5.0 ([#8576](https://github.com/opensearch-project/OpenSearch/pull/8576)) - Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) - Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) +- Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b1a87fe6c3112..b6dc8820a85c5 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -55,7 +55,7 @@ dependencies { api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" - api 'net.minidev:json-smart:2.4.11' + api 'net.minidev:json-smart:2.5.0' api "org.mockito:mockito-core:${versions.mockito}" api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java index 5e111364b955e..99c62d1d0b707 100644 --- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java +++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java @@ -32,6 +32,17 @@ package hdfs; +import java.io.File; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -46,18 +57,6 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.security.UserGroupInformation; -import java.io.File; -import java.lang.management.ManagementFactory; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - /** * MiniHDFS test fixture. There is a CLI tool, but here we can * easily properly setup logging, avoid parsing JSON, etc. @@ -74,7 +73,6 @@ public static void main(String[] args) throws Exception { } boolean secure = args.length == 3; - // configure Paths Path baseDir = Paths.get(args[0]); // hadoop-home/, so logs will not complain if (System.getenv("HADOOP_HOME") == null) { From c677f355589ead8b5d9458ef75e82749dba44d76 Mon Sep 17 00:00:00 2001 From: Harish Bhakuni Date: Tue, 11 Jul 2023 00:45:22 -0700 Subject: [PATCH 096/186] [Snapshot Interop] Keep API parameters behind remote store experimental flag. (#8594) Signed-off-by: Harish Bhakuni --- .../restore/RestoreSnapshotRequest.java | 69 +++++++++++++------ .../repositories/RepositoriesService.java | 7 ++ .../opensearch/snapshots/RestoreService.java | 6 ++ 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 9ab66d726854e..cddb338310c4f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -41,6 +41,7 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -150,7 +151,7 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_7_0)) { storageType = in.readEnum(StorageType.class); } - if (in.getVersion().onOrAfter(Version.V_2_9_0)) { + if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE) && in.getVersion().onOrAfter(Version.V_2_9_0)) { sourceRemoteStoreRepository = in.readOptionalString(); } } @@ -174,7 +175,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_7_0)) { out.writeEnum(storageType); } - if (out.getVersion().onOrAfter(Version.V_2_9_0)) { + if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE) && out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeOptionalString(sourceRemoteStoreRepository); } } @@ -614,6 +615,11 @@ public RestoreSnapshotRequest source(Map source) { } } else if (name.equals("source_remote_store_repository")) { + if (!FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE)) { + throw new IllegalArgumentException( + "Unsupported parameter " + name + ". Please enable remote store feature flag for this experimental feature" + ); + } if (entry.getValue() instanceof String) { setSourceRemoteStoreRepository((String) entry.getValue()); } else { @@ -664,7 +670,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (storageType != null) { storageType.toXContent(builder); } - if (sourceRemoteStoreRepository != null) { + if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE) && sourceRemoteStoreRepository != null) { builder.field("source_remote_store_repository", sourceRemoteStoreRepository); } builder.endObject(); @@ -681,7 +687,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RestoreSnapshotRequest that = (RestoreSnapshotRequest) o; - return waitForCompletion == that.waitForCompletion + boolean equals = waitForCompletion == that.waitForCompletion && includeGlobalState == that.includeGlobalState && partial == that.partial && includeAliases == that.includeAliases @@ -694,27 +700,48 @@ public boolean equals(Object o) { && Objects.equals(indexSettings, that.indexSettings) && Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings) && Objects.equals(snapshotUuid, that.snapshotUuid) - && Objects.equals(storageType, that.storageType) - && Objects.equals(sourceRemoteStoreRepository, that.sourceRemoteStoreRepository); + && Objects.equals(storageType, that.storageType); + if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE)) { + equals = Objects.equals(sourceRemoteStoreRepository, that.sourceRemoteStoreRepository); + } + return equals; } @Override public int hashCode() { - int result = Objects.hash( - snapshot, - repository, - indicesOptions, - renamePattern, - renameReplacement, - waitForCompletion, - includeGlobalState, - partial, - includeAliases, - indexSettings, - snapshotUuid, - storageType, - sourceRemoteStoreRepository - ); + int result; + if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE)) { + result = Objects.hash( + snapshot, + repository, + indicesOptions, + renamePattern, + renameReplacement, + waitForCompletion, + includeGlobalState, + partial, + includeAliases, + indexSettings, + snapshotUuid, + storageType, + sourceRemoteStoreRepository + ); + } else { + result = Objects.hash( + snapshot, + repository, + indicesOptions, + renamePattern, + renameReplacement, + waitForCompletion, + includeGlobalState, + partial, + includeAliases, + indexSettings, + snapshotUuid, + storageType + ); + } result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(ignoreIndexSettings); return result; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 9c56d172f2ea1..e7f7a1d9c0554 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -63,6 +63,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; @@ -627,6 +628,12 @@ public static void validateRepositoryMetadataSettings( + minVersionInCluster ); } + if (REMOTE_STORE_INDEX_SHALLOW_COPY.get(repositoryMetadataSettings) && !FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE)) { + throw new RepositoryException( + repositoryName, + "setting " + REMOTE_STORE_INDEX_SHALLOW_COPY.getKey() + " cannot be enabled, as remote store feature is not enabled." + ); + } } private static void ensureRepositoryNotInUse(ClusterState clusterState, String repository) { diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index ebd0e59599c21..7df24e5357555 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -453,6 +453,12 @@ public ClusterState execute(ClusterState currentState) { final boolean isRemoteStoreShallowCopy = Boolean.TRUE.equals( snapshotInfo.isRemoteStoreIndexShallowCopyEnabled() ) && metadata.index(index).getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false); + if (isSearchableSnapshot && isRemoteStoreShallowCopy) { + throw new SnapshotRestoreException( + snapshot, + "Shallow copy snapshot cannot be restored as searchable snapshot." + ); + } if (isRemoteStoreShallowCopy && !currentState.getNodes().getMinNodeVersion().onOrAfter(Version.V_2_9_0)) { throw new SnapshotRestoreException( snapshot, From a430e18e82f7671e01c37ae5a549fc36dd673b1a Mon Sep 17 00:00:00 2001 From: Bansi Kasundra <66969140+kasundra07@users.noreply.github.com> Date: Tue, 11 Jul 2023 01:02:32 -0700 Subject: [PATCH 097/186] Add Changes in Snapshot Delete Flow for remote store interoperability. (#7497) Signed-off-by: Bansi Kasundra --- .../snapshots/DeleteSnapshotIT.java | 305 ++++++++++++++++++ .../TransportCleanupRepositoryAction.java | 5 + .../opensearch/repositories/Repository.java | 19 ++ .../blobstore/BlobStoreRepository.java | 273 ++++++++++++---- .../snapshots/SnapshotsService.java | 24 +- 5 files changed, 569 insertions(+), 57 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java new file mode 100644 index 0000000000000..2688449294f3d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -0,0 +1,305 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import org.opensearch.action.ActionFuture; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.client.Client; +import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.is; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase { + + public void testDeleteSnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "fs", snapshotRepoPath); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String snapshot = "snapshot"; + createFullSnapshot(snapshotRepoName, snapshot); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 1); + + assertAcked(startDeleteSnapshot(snapshotRepoName, snapshot).get()); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); + } + + public void testDeleteShallowCopySnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String shallowSnapshot = "shallow-snapshot"; + createFullSnapshot(snapshotRepoName, shallowSnapshot); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 1); + + assertAcked(startDeleteSnapshot(snapshotRepoName, shallowSnapshot).get()); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + } + + // Deleting multiple shallow copy snapshots as part of single delete call with repo having only shallow copy snapshots. + public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(2); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); + final String testIndex = "index-test"; + createIndexWithContent(testIndex); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + // Creating some shallow copy snapshots + int totalShallowCopySnapshotsCount = randomIntBetween(4, 10); + List shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount); + List snapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount)); + int tobeDeletedSnapshotsCount = snapshotsToBeDeleted.size(); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount); + // Deleting subset of shallow copy snapshots + assertAcked( + clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, snapshotsToBeDeleted.toArray(new String[0])) + .get() + ); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount + - tobeDeletedSnapshotsCount); + } + + // Deleting multiple shallow copy snapshots as part of single delete call with both partial and full copy snapshot present in the repo + // And then deleting multiple full copy snapshots as part of single delete call with both partial and shallow copy snapshots present in + // the repo + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8610") + public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + + internalCluster().startClusterManagerOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + ensureStableCluster(2); + final String clusterManagerNode = internalCluster().getClusterManagerName(); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); + final String testIndex = "index-test"; + createIndexWithContent(testIndex); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + // Creating a partial shallow copy snapshot + final String snapshot = "snapshot"; + blockNodeWithIndex(snapshotRepoName, testIndex); + blockDataNode(snapshotRepoName, dataNode); + + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + final ActionFuture snapshotFuture = clusterManagerClient.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshot) + .setWaitForCompletion(true) + .execute(); + + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(dataNode, snapshotRepoName, TimeValue.timeValueSeconds(30L)); + internalCluster().restartNode(dataNode); + assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); + + unblockAllDataNodes(snapshotRepoName); + + ensureStableCluster(2, clusterManagerNode); + + // Creating some shallow copy snapshots + int totalShallowCopySnapshotsCount = randomIntBetween(4, 10); + List shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount); + List shallowCopySnapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount)); + int tobeDeletedShallowCopySnapshotsCount = shallowCopySnapshotsToBeDeleted.size(); + totalShallowCopySnapshotsCount += 1; // Adding partial shallow snapshot here + // Updating the snapshot repository flag to disable shallow snapshots + createRepository(snapshotRepoName, "mock", snapshotRepoPath); + // Creating some full copy snapshots + int totalFullCopySnapshotsCount = randomIntBetween(4, 10); + List fullCopySnapshots = createNSnapshots(snapshotRepoName, totalFullCopySnapshotsCount); + List fullCopySnapshotsToBeDeleted = fullCopySnapshots.subList(0, randomIntBetween(2, totalFullCopySnapshotsCount)); + int tobeDeletedFullCopySnapshotsCount = fullCopySnapshotsToBeDeleted.size(); + + int totalSnapshotsCount = totalFullCopySnapshotsCount + totalShallowCopySnapshotsCount; + + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); + // Deleting subset of shallow copy snapshots + assertAcked( + clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, shallowCopySnapshotsToBeDeleted.toArray(new String[0])) + .get() + ); + totalSnapshotsCount -= tobeDeletedShallowCopySnapshotsCount; + totalShallowCopySnapshotsCount -= tobeDeletedShallowCopySnapshotsCount; + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + + // Deleting subset of full copy snapshots + assertAcked( + clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, fullCopySnapshotsToBeDeleted.toArray(new String[0])) + .get() + ); + totalSnapshotsCount -= tobeDeletedFullCopySnapshotsCount; + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + } + + // Deleting subset of shallow and full copy snapshots as part of single delete call and then deleting all snapshots in the repo. + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8610") + public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(2); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); + final String testIndex = "index-test"; + createIndexWithContent(testIndex); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + // Creating some shallow copy snapshots + int totalShallowCopySnapshotsCount = randomIntBetween(4, 10); + List shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount); + List shallowCopySnapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount)); + int tobeDeletedShallowCopySnapshotsCount = shallowCopySnapshotsToBeDeleted.size(); + // Updating the snapshot repository flag to disable shallow snapshots + createRepository(snapshotRepoName, "mock", snapshotRepoPath); + // Creating some full copy snapshots + int totalFullCopySnapshotsCount = randomIntBetween(4, 10); + List fullCopySnapshots = createNSnapshots(snapshotRepoName, totalFullCopySnapshotsCount); + List fullCopySnapshotsToBeDeleted = fullCopySnapshots.subList(0, randomIntBetween(2, totalFullCopySnapshotsCount)); + int tobeDeletedFullCopySnapshotsCount = fullCopySnapshotsToBeDeleted.size(); + + int totalSnapshotsCount = totalFullCopySnapshotsCount + totalShallowCopySnapshotsCount; + + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); + // Deleting subset of shallow copy snapshots and full copy snapshots + assertAcked( + clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot( + snapshotRepoName, + Stream.concat(shallowCopySnapshotsToBeDeleted.stream(), fullCopySnapshotsToBeDeleted.stream()).toArray(String[]::new) + ) + .get() + ); + totalSnapshotsCount -= (tobeDeletedShallowCopySnapshotsCount + tobeDeletedFullCopySnapshotsCount); + totalShallowCopySnapshotsCount -= tobeDeletedShallowCopySnapshotsCount; + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + + // Deleting all the remaining snapshots + assertAcked(clusterManagerClient.admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "*").get()); + assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + } + + private List createNSnapshots(String repoName, int count) { + final List snapshotNames = new ArrayList<>(count); + final String prefix = "snap-" + UUIDs.randomBase64UUID(random()).toLowerCase(Locale.ROOT) + "-"; + for (int i = 0; i < count; i++) { + final String name = prefix + i; + createFullSnapshot(repoName, name); + snapshotNames.add(name); + } + logger.info("--> created {} in [{}]", snapshotNames, repoName); + return snapshotNames; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index 07b918e427784..ec02173ff2807 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -52,6 +52,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryCleanupResult; @@ -93,6 +94,8 @@ public final class TransportCleanupRepositoryAction extends TransportClusterMana private final SnapshotsService snapshotsService; + private final RemoteStoreLockManagerFactory remoteStoreLockManagerFactory; + @Override protected String executor() { return ThreadPool.Names.SAME; @@ -119,6 +122,7 @@ public TransportCleanupRepositoryAction( ); this.repositoriesService = repositoriesService; this.snapshotsService = snapshotsService; + this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(() -> repositoriesService); // We add a state applier that will remove any dangling repository cleanup actions on cluster-manager failover. // This is safe to do since cleanups will increment the repository state id before executing any operations to prevent concurrent // operations from corrupting the repository. This is the same safety mechanism used by snapshot deletes. @@ -267,6 +271,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS l -> blobStoreRepository.cleanup( repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), + remoteStoreLockManagerFactory, ActionListener.wrap(result -> after(null, result), e -> after(e, null)) ) ) diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 793b3d317e1bd..4367b4c65851e 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -169,6 +169,25 @@ void deleteSnapshots( ActionListener listener ); + /** + * Deletes snapshots and releases respective lock files from remote store repository. + * + * @param snapshotIds snapshot ids + * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began + * @param repositoryMetaVersion version of the updated repository metadata to write + * @param remoteStoreLockManagerFactory RemoteStoreLockManagerFactory to be used for cleaning up remote store lock files + * @param listener completion listener + */ + default void deleteSnapshotsAndReleaseLockFiles( + Collection snapshotIds, + long repositoryStateId, + Version repositoryMetaVersion, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + throw new UnsupportedOperationException(); + } + /** * Returns snapshot throttle time in nanoseconds */ diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 3e77a7e796375..398d78392ae17 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -797,11 +797,11 @@ public RepositoryStats stats() { return new RepositoryStats(store.stats()); } - @Override - public void deleteSnapshots( + public void deleteSnapshotsAndReleaseLockFiles( Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, ActionListener listener ) { if (isReadOnly()) { @@ -822,6 +822,7 @@ protected void doRun() throws Exception { rootBlobs, repositoryData, repositoryMetaVersion, + remoteStoreLockManagerFactory, listener ); } @@ -834,6 +835,22 @@ public void onFailure(Exception e) { } } + @Override + public void deleteSnapshots( + Collection snapshotIds, + long repositoryStateId, + Version repositoryMetaVersion, + ActionListener listener + ) { + deleteSnapshotsAndReleaseLockFiles( + snapshotIds, + repositoryStateId, + repositoryMetaVersion, + null, // Passing null since no remote store lock files need to be cleaned up. + listener + ); + } + /** * Loads {@link RepositoryData} ensuring that it is consistent with the given {@code rootBlobs} as well of the assumed generation. * @@ -883,16 +900,18 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map snapshotIds, @@ -901,11 +920,18 @@ private void doDeleteShardSnapshots( Map rootBlobs, RepositoryData repositoryData, Version repoMetaVersion, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, ActionListener listener ) { // First write the new shard state metadata (with the removed snapshot) and compute deletion targets final StepListener> writeShardMetaDataAndComputeDeletesStep = new StepListener<>(); - writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, true, writeShardMetaDataAndComputeDeletesStep); + writeUpdatedShardMetaDataAndComputeDeletes( + snapshotIds, + repositoryData, + true, + remoteStoreLockManagerFactory, + writeShardMetaDataAndComputeDeletesStep + ); // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: // 1. Remove the snapshots from the list of existing snapshots // 2. Update the index shard generations of all updated shard folders @@ -935,11 +961,19 @@ private void doDeleteShardSnapshots( ActionListener.wrap(() -> listener.onResponse(updatedRepoData)), 2 ); - cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); + cleanupUnlinkedRootAndIndicesBlobs( + snapshotIds, + foundIndices, + rootBlobs, + updatedRepoData, + remoteStoreLockManagerFactory, + afterCleanupsListener + ); asyncCleanupUnlinkedShardLevelBlobs( repositoryData, snapshotIds, writeShardMetaDataAndComputeDeletesStep.result(), + remoteStoreLockManagerFactory, afterCleanupsListener ); }, listener::onFailure); @@ -950,15 +984,24 @@ private void cleanupUnlinkedRootAndIndicesBlobs( Map foundIndices, Map rootBlobs, RepositoryData updatedRepoData, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, ActionListener listener ) { - cleanupStaleBlobs(deletedSnapshots, foundIndices, rootBlobs, updatedRepoData, ActionListener.map(listener, ignored -> null)); + cleanupStaleBlobs( + deletedSnapshots, + foundIndices, + rootBlobs, + updatedRepoData, + remoteStoreLockManagerFactory, + ActionListener.map(listener, ignored -> null) + ); } private void asyncCleanupUnlinkedShardLevelBlobs( RepositoryData oldRepositoryData, Collection snapshotIds, Collection deleteResults, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, ActionListener listener ) { final List filesToDelete = resolveFilesToDelete(oldRepositoryData, snapshotIds, deleteResults); @@ -982,7 +1025,7 @@ private void asyncCleanupUnlinkedShardLevelBlobs( // Start as many workers as fit into the snapshot pool at once at the most final int workers = Math.min(threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(), staleFilesToDeleteInBatch.size()); for (int i = 0; i < workers; ++i) { - executeStaleShardDelete(staleFilesToDeleteInBatch, groupedListener); + executeStaleShardDelete(staleFilesToDeleteInBatch, remoteStoreLockManagerFactory, groupedListener); } } catch (Exception e) { @@ -995,12 +1038,49 @@ private void asyncCleanupUnlinkedShardLevelBlobs( } } - private void executeStaleShardDelete(BlockingQueue> staleFilesToDeleteInBatch, GroupedActionListener listener) - throws InterruptedException { + // When remoteStoreLockManagerFactory is non-null, while deleting the files, lock files are also released before deletion of respective + // shallow-snap-UUID files. And if it is null, we just delete the stale shard blobs. + private void executeStaleShardDelete( + BlockingQueue> staleFilesToDeleteInBatch, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + GroupedActionListener listener + ) throws InterruptedException { List filesToDelete = staleFilesToDeleteInBatch.poll(0L, TimeUnit.MILLISECONDS); if (filesToDelete != null) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> { try { + if (remoteStoreLockManagerFactory != null) { + for (String fileToDelete : filesToDelete) { + if (fileToDelete.contains(SHALLOW_SNAPSHOT_PREFIX)) { + String[] fileToDeletePath = fileToDelete.split("/"); + String indexId = fileToDeletePath[1]; + String shardId = fileToDeletePath[2]; + String shallowSnapBlob = fileToDeletePath[3]; + String snapshotUUID = shallowSnapBlob.substring( + SHALLOW_SNAPSHOT_PREFIX.length(), + shallowSnapBlob.length() - ".dat".length() + ); + BlobContainer shardContainer = blobStore().blobContainer(indicesPath().add(indexId).add(shardId)); + RemoteStoreShardShallowCopySnapshot remoteStoreShardShallowCopySnapshot = + REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read( + shardContainer, + snapshotUUID, + namedXContentRegistry + ); + String indexUUID = remoteStoreShardShallowCopySnapshot.getIndexUUID(); + String remoteStoreRepoForIndex = remoteStoreShardShallowCopySnapshot.getRemoteStoreRepository(); + // Releasing lock file before deleting the shallow-snap-UUID file because in case of any failure while + // releasing the lock file, we would still have the shallow-snap-UUID file and that would be used during + // next delete operation for releasing this lock file + RemoteStoreMetadataLockManager remoteStoreMetadataLockManager = remoteStoreLockManagerFactory + .newLockManager(remoteStoreRepoForIndex, indexUUID, shardId); + remoteStoreMetadataLockManager.release( + FileLockInfo.getLockInfoBuilder().withAcquirerId(snapshotUUID).build() + ); + } + } + } + // Deleting the shard blobs deleteFromContainer(blobContainer(), filesToDelete); l.onResponse(null); } catch (Exception e) { @@ -1014,7 +1094,7 @@ private void executeStaleShardDelete(BlockingQueue> staleFilesToDel ); l.onFailure(e); } - executeStaleShardDelete(staleFilesToDeleteInBatch, listener); + executeStaleShardDelete(staleFilesToDeleteInBatch, remoteStoreLockManagerFactory, listener); })); } } @@ -1024,6 +1104,7 @@ private void writeUpdatedShardMetaDataAndComputeDeletes( Collection snapshotIds, RepositoryData oldRepositoryData, boolean useUUIDs, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, ActionListener> onAllShardsCompleted ) { @@ -1096,17 +1177,30 @@ protected void doRun() throws Exception { final Set blobs = shardContainer.listBlobs().keySet(); final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; final long newGen; - if (useUUIDs) { - newGen = -1L; - blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots( - blobs, - shardContainer, - oldRepositoryData.shardGenerations().getShardGen(indexId, finalShardId) - ).v1(); + + // Index-N file would be present if snapshots other than shallow snapshots are present for this shard + if (blobs.stream() + .filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)) + .collect(Collectors.toSet()) + .size() > 0) { + if (useUUIDs) { + newGen = -1L; + blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots( + blobs, + shardContainer, + oldRepositoryData.shardGenerations().getShardGen(indexId, finalShardId) + ).v1(); + } else { + Tuple tuple = buildBlobStoreIndexShardSnapshots( + blobs, + shardContainer + ); + newGen = tuple.v2() + 1; + blobStoreIndexShardSnapshots = tuple.v1(); + } } else { - Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer); - newGen = tuple.v2() + 1; - blobStoreIndexShardSnapshots = tuple.v1(); + newGen = -1L; + blobStoreIndexShardSnapshots = BlobStoreIndexShardSnapshots.EMPTY; } allShardsListener.onResponse( deleteFromShardSnapshotMeta( @@ -1117,7 +1211,8 @@ protected void doRun() throws Exception { shardContainer, blobs, blobStoreIndexShardSnapshots, - newGen + newGen, + remoteStoreLockManagerFactory ) ); } @@ -1168,20 +1263,23 @@ private List resolveFilesToDelete( /** * Cleans up stale blobs directly under the repository root as well as all indices paths that aren't referenced by any existing * snapshots. This method is only to be called directly after a new {@link RepositoryData} was written to the repository and with - * parameters {@code foundIndices}, {@code rootBlobs} + * parameters {@code foundIndices}, {@code rootBlobs}. If remoteStoreLockManagerFactory is not null, remote store lock files are + * released when deleting the respective shallow-snap-UUID blobs. * - * @param deletedSnapshots if this method is called as part of a delete operation, the snapshot ids just deleted or empty if called as - * part of a repository cleanup - * @param foundIndices all indices blob containers found in the repository before {@code newRepoData} was written - * @param rootBlobs all blobs found directly under the repository root - * @param newRepoData new repository data that was just written - * @param listener listener to invoke with the combined {@link DeleteResult} of all blobs removed in this operation + * @param deletedSnapshots if this method is called as part of a delete operation, the snapshot ids just deleted or empty if called as + * part of a repository cleanup + * @param foundIndices all indices blob containers found in the repository before {@code newRepoData} was written + * @param rootBlobs all blobs found directly under the repository root + * @param newRepoData new repository data that was just written + * @param remoteStoreLockManagerFactory RemoteStoreLockManagerFactory to be used for cleaning up remote store lock files. + * @param listener listener to invoke with the combined {@link DeleteResult} of all blobs removed in this operation */ private void cleanupStaleBlobs( Collection deletedSnapshots, Map foundIndices, Map rootBlobs, RepositoryData newRepoData, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, ActionListener listener ) { final GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(deleteResults -> { @@ -1207,24 +1305,31 @@ private void cleanupStaleBlobs( if (foundIndices.keySet().equals(survivingIndexIds)) { groupedListener.onResponse(DeleteResult.ZERO); } else { - cleanupStaleIndices(foundIndices, survivingIndexIds, groupedListener); + cleanupStaleIndices(foundIndices, survivingIndexIds, remoteStoreLockManagerFactory, groupedListener); } } /** * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the - * repository. + * repository. If remoteStoreLockManagerFactory is not null, remote store lock files are released when deleting the respective + * shallow-snap-UUID blobs. * TODO: Add shard level cleanups * TODO: Add unreferenced index metadata cleanup *

    *
  • Deleting stale indices {@link #cleanupStaleIndices}
    • *
  • Deleting unreferenced root level blobs {@link #cleanupStaleRootFiles}
    • *
- * @param repositoryStateId Current repository state id - * @param repositoryMetaVersion version of the updated repository metadata to write - * @param listener Listener to complete when done + * @param repositoryStateId Current repository state id + * @param repositoryMetaVersion version of the updated repository metadata to write + * @param remoteStoreLockManagerFactory RemoteStoreLockManagerFactory to be used for cleaning up remote store lock files. + * @param listener Listener to complete when done */ - public void cleanup(long repositoryStateId, Version repositoryMetaVersion, ActionListener listener) { + public void cleanup( + long repositoryStateId, + Version repositoryMetaVersion, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { try { if (isReadOnly()) { throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository"); @@ -1254,6 +1359,7 @@ public void cleanup(long repositoryStateId, Version repositoryMetaVersion, Actio foundIndices, rootBlobs, repositoryData, + remoteStoreLockManagerFactory, ActionListener.map(listener, RepositoryCleanupResult::new) ), listener::onFailure @@ -1345,6 +1451,7 @@ private List cleanupStaleRootFiles( private void cleanupStaleIndices( Map foundIndices, Set survivingIndexIds, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, GroupedActionListener listener ) { final GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(deleteResults -> { @@ -1369,7 +1476,7 @@ private void cleanupStaleIndices( foundIndices.size() - survivingIndexIds.size() ); for (int i = 0; i < workers; ++i) { - executeOneStaleIndexDelete(staleIndicesToDelete, groupedListener); + executeOneStaleIndexDelete(staleIndicesToDelete, remoteStoreLockManagerFactory, groupedListener); } } catch (Exception e) { // TODO: We shouldn't be blanket catching and suppressing all exceptions here and instead handle them safely upstream. @@ -1382,6 +1489,7 @@ private void cleanupStaleIndices( private void executeOneStaleIndexDelete( BlockingQueue> staleIndicesToDelete, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, GroupedActionListener listener ) throws InterruptedException { Map.Entry indexEntry = staleIndicesToDelete.poll(0L, TimeUnit.MILLISECONDS); @@ -1391,6 +1499,37 @@ private void executeOneStaleIndexDelete( DeleteResult deleteResult = DeleteResult.ZERO; try { logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId); + if (remoteStoreLockManagerFactory != null) { + Map shardBlobs = indexEntry.getValue().children(); + if (!shardBlobs.isEmpty()) { + for (Map.Entry shardBlob : shardBlobs.entrySet()) { + Map shardLevelBlobs = shardBlob.getValue().listBlobs(); + for (Map.Entry shardLevelBlob : shardLevelBlobs.entrySet()) { + String blob = shardLevelBlob.getKey(); + String snapshotUUID = blob.substring(SHALLOW_SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()); + if (blob.startsWith(SHALLOW_SNAPSHOT_PREFIX) && blob.endsWith(".dat")) { + RemoteStoreShardShallowCopySnapshot remoteStoreShardShallowCopySnapshot = + REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read( + shardBlob.getValue(), + snapshotUUID, + namedXContentRegistry + ); + String indexUUID = remoteStoreShardShallowCopySnapshot.getIndexUUID(); + String remoteStoreRepoForIndex = remoteStoreShardShallowCopySnapshot.getRemoteStoreRepository(); + // Releasing lock files before deleting the shallow-snap-UUID file because in case of any failure + // while releasing the lock file, we would still have the corresponding shallow-snap-UUID file + // and that would be used during next delete operation for releasing this stale lock file + RemoteStoreMetadataLockManager remoteStoreMetadataLockManager = remoteStoreLockManagerFactory + .newLockManager(remoteStoreRepoForIndex, indexUUID, shardBlob.getKey()); + remoteStoreMetadataLockManager.release( + FileLockInfo.getLockInfoBuilder().withAcquirerId(snapshotUUID).build() + ); + } + } + } + } + } + // Deleting the index folder deleteResult = indexEntry.getValue().delete(); logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); } catch (IOException e) { @@ -1408,7 +1547,7 @@ private void executeOneStaleIndexDelete( logger.warn(new ParameterizedMessage("[{}] Exception during single stale index delete", metadata.name()), e); } - executeOneStaleIndexDelete(staleIndicesToDelete, listener); + executeOneStaleIndexDelete(staleIndicesToDelete, remoteStoreLockManagerFactory, listener); return deleteResult; })); } @@ -2942,7 +3081,8 @@ private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta( BlobContainer shardContainer, Set blobs, BlobStoreIndexShardSnapshots snapshots, - long indexGeneration + long indexGeneration, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory ) { // Build a list of snapshots that should be preserved List newSnapshotsList = new ArrayList<>(); @@ -2954,23 +3094,37 @@ private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta( } String writtenGeneration = null; try { - if (newSnapshotsList.isEmpty()) { + // Using survivingSnapshots instead of newSnapshotsList as shallow snapshots can be present which won't be part of + // newSnapshotsList + if (survivingSnapshots.isEmpty()) { + // No shallow copy or full copy snapshot is surviving. return new ShardSnapshotMetaDeleteResult(indexId, snapshotShardId, ShardGenerations.DELETED_SHARD_GEN, blobs); } else { - final BlobStoreIndexShardSnapshots updatedSnapshots = new BlobStoreIndexShardSnapshots(newSnapshotsList); - if (indexGeneration < 0L) { - writtenGeneration = UUIDs.randomBase64UUID(); - INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedSnapshots, shardContainer, writtenGeneration, compressor); + final BlobStoreIndexShardSnapshots updatedSnapshots; + // If we have surviving non shallow snapshots, update index- file. + if (newSnapshotsList.size() > 0) { + // Some full copy snapshots are surviving. + updatedSnapshots = new BlobStoreIndexShardSnapshots(newSnapshotsList); + if (indexGeneration < 0L) { + writtenGeneration = UUIDs.randomBase64UUID(); + INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedSnapshots, shardContainer, writtenGeneration, compressor); + } else { + writtenGeneration = String.valueOf(indexGeneration); + writeShardIndexBlobAtomic(shardContainer, indexGeneration, updatedSnapshots); + } } else { - writtenGeneration = String.valueOf(indexGeneration); - writeShardIndexBlobAtomic(shardContainer, indexGeneration, updatedSnapshots); + // Some shallow copy snapshots are surviving. In this case, since no full copy snapshots are present, we use + // EMPTY BlobStoreIndexShardSnapshots for updatedSnapshots which is used in unusedBlobs to compute stale files, + // and use DELETED_SHARD_GEN since index-N file would not be present anymore. + updatedSnapshots = BlobStoreIndexShardSnapshots.EMPTY; + writtenGeneration = ShardGenerations.DELETED_SHARD_GEN; } final Set survivingSnapshotUUIDs = survivingSnapshots.stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); return new ShardSnapshotMetaDeleteResult( indexId, snapshotShardId, writtenGeneration, - unusedBlobs(blobs, survivingSnapshotUUIDs, updatedSnapshots) + unusedBlobs(blobs, survivingSnapshotUUIDs, updatedSnapshots, remoteStoreLockManagerFactory) ); } } catch (IOException e) { @@ -3004,11 +3158,13 @@ private void writeShardIndexBlobAtomic( } // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all - // temporary blobs + // temporary blobs. If remoteStoreLockManagerFactory is non-null, the shallow-snap- files that do not belong to any of the + // surviving snapshots are also added for cleanup. private static List unusedBlobs( Set blobs, Set survivingSnapshotUUIDs, - BlobStoreIndexShardSnapshots updatedSnapshots + BlobStoreIndexShardSnapshots updatedSnapshots, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory ) { return blobs.stream() .filter( @@ -3018,6 +3174,13 @@ private static List unusedBlobs( && survivingSnapshotUUIDs.contains( blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) ) == false) + || (remoteStoreLockManagerFactory != null + ? (blob.startsWith(SHALLOW_SNAPSHOT_PREFIX) + && blob.endsWith(".dat") + && survivingSnapshotUUIDs.contains( + blob.substring(SHALLOW_SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) + ) == false) + : false) || (blob.startsWith(UPLOADED_DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null) || FsBlobContainer.isTempBlobName(blob) ) diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 0a455c376f62d..3946e4363298c 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -2230,16 +2230,36 @@ private void deleteSnapshotsFromRepository( assert currentlyFinalizing.contains(deleteEntry.repository()); final List snapshotIds = deleteEntry.getSnapshots(); assert deleteEntry.state() == SnapshotDeletionsInProgress.State.STARTED : "incorrect state for entry [" + deleteEntry + "]"; - repositoriesService.repository(deleteEntry.repository()) - .deleteSnapshots( + final Repository repository = repositoriesService.repository(deleteEntry.repository()); + + // TODO: Relying on repository flag to decide delete flow may lead to shallow snapshot blobs not being taken up for cleanup + // when the repository currently have the flag disabled and we try to delete the shallow snapshots taken prior to disabling + // the flag. This can be improved by having the info whether there ever were any shallow snapshot present in this repository + // or not in RepositoryData. + // SEE https://github.com/opensearch-project/OpenSearch/issues/8610 + final boolean cleanupRemoteStoreLockFiles = REMOTE_STORE_INDEX_SHALLOW_COPY.get(repository.getMetadata().settings()); + if (cleanupRemoteStoreLockFiles) { + repository.deleteSnapshotsAndReleaseLockFiles( snapshotIds, repositoryData.getGenId(), minCompatibleVersion(minNodeVersion, repositoryData, snapshotIds), + remoteStoreLockManagerFactory, ActionListener.wrap(updatedRepoData -> { logger.info("snapshots {} deleted", snapshotIds); removeSnapshotDeletionFromClusterState(deleteEntry, null, updatedRepoData); }, ex -> removeSnapshotDeletionFromClusterState(deleteEntry, ex, repositoryData)) ); + } else { + repository.deleteSnapshots( + snapshotIds, + repositoryData.getGenId(), + minCompatibleVersion(minNodeVersion, repositoryData, snapshotIds), + ActionListener.wrap(updatedRepoData -> { + logger.info("snapshots {} deleted", snapshotIds); + removeSnapshotDeletionFromClusterState(deleteEntry, null, updatedRepoData); + }, ex -> removeSnapshotDeletionFromClusterState(deleteEntry, ex, repositoryData)) + ); + } } } From 8a831f84917ada0a9ad723691469024e61be1d33 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Tue, 11 Jul 2023 17:07:10 +0530 Subject: [PATCH 098/186] Add shard id to remote store logs (#8574) --------- Signed-off-by: bansvaru --- .../opensearch/index/shard/IndexShard.java | 2 +- .../shard/RemoteStoreRefreshListener.java | 13 ++++-- .../index/translog/RemoteFsTranslog.java | 43 ++++++++++++------- .../transfer/TranslogTransferManager.java | 16 +++---- .../TranslogTransferManagerTests.java | 2 + 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index e7720e9343b80..154e1a4f22242 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4589,7 +4589,7 @@ public void syncTranslogFilesFromRemoteTranslog() throws IOException { TranslogFactory translogFactory = translogFactorySupplier.apply(indexSettings, shardRouting); assert translogFactory instanceof RemoteBlobStoreInternalTranslogFactory; Repository repository = ((RemoteBlobStoreInternalTranslogFactory) translogFactory).getRepository(); - RemoteFsTranslog.download(repository, shardId, getThreadPool(), shardPath().resolveTranslog()); + RemoteFsTranslog.download(repository, shardId, getThreadPool(), shardPath().resolveTranslog(), logger); } /** diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index aaba74cd54341..46d52bc8ca5df 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -8,7 +8,6 @@ package org.opensearch.index.shard; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; @@ -22,6 +21,7 @@ import org.opensearch.action.bulk.BackoffPolicy; import org.opensearch.common.CheckedFunction; import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.logging.Loggers; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.engine.EngineException; @@ -60,7 +60,7 @@ */ public final class RemoteStoreRefreshListener implements ReferenceManager.RefreshListener { - private static final Logger logger = LogManager.getLogger(RemoteStoreRefreshListener.class); + private final Logger logger; /** * The initial retry interval at which the retry job gets scheduled after a failure. @@ -117,6 +117,7 @@ public RemoteStoreRefreshListener( SegmentReplicationCheckpointPublisher checkpointPublisher, RemoteRefreshSegmentTracker segmentTracker ) { + logger = Loggers.getLogger(getClass(), indexShard.shardId()); this.indexShard = indexShard; this.storeDirectory = indexShard.store().directory(); this.remoteDirectory = (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()) @@ -155,7 +156,7 @@ public void onFailure(String file) { // Track upload failure segmentTracker.addUploadBytesFailed(latestFileNameSizeOnLocalMap.get(file)); } - }, remoteDirectory, storeDirectory, this::getChecksumOfLocalFile); + }, remoteDirectory, storeDirectory, this::getChecksumOfLocalFile, logger); } @Override @@ -470,6 +471,8 @@ private void updateFinalUploadStatusInSegmentTracker(boolean uploadStatus, long */ private static class FileUploader { + private final Logger logger; + private final UploadTracker uploadTracker; private final RemoteSegmentStoreDirectory remoteDirectory; @@ -482,12 +485,14 @@ public FileUploader( UploadTracker uploadTracker, RemoteSegmentStoreDirectory remoteDirectory, Directory storeDirectory, - CheckedFunction checksumProvider + CheckedFunction checksumProvider, + Logger logger ) { this.uploadTracker = uploadTracker; this.remoteDirectory = remoteDirectory; this.storeDirectory = storeDirectory; this.checksumProvider = checksumProvider; + this.logger = logger; } /** diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 1e565b97387d1..9e027b9765bbc 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -8,9 +8,9 @@ package org.opensearch.index.translog; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.SetOnce; +import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.util.FileSystemUtils; @@ -32,6 +32,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.Semaphore; @@ -49,7 +50,7 @@ */ public class RemoteFsTranslog extends Translog { - private static final Logger logger = LogManager.getLogger(RemoteFsTranslog.class); + private final Logger logger; private final BlobStoreRepository blobStoreRepository; private final TranslogTransferManager translogTransferManager; private final FileTransferTracker fileTransferTracker; @@ -82,16 +83,19 @@ public RemoteFsTranslog( BooleanSupplier primaryModeSupplier ) throws IOException { super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer); + logger = Loggers.getLogger(getClass(), shardId); this.blobStoreRepository = blobStoreRepository; this.primaryModeSupplier = primaryModeSupplier; fileTransferTracker = new FileTransferTracker(shardId); this.translogTransferManager = buildTranslogTransferManager(blobStoreRepository, threadPool, shardId, fileTransferTracker); try { - download(translogTransferManager, location); + download(translogTransferManager, location, logger); Checkpoint checkpoint = readCheckpoint(location); this.readers.addAll(recoverFromFiles(checkpoint)); if (readers.isEmpty()) { - throw new IllegalStateException("at least one reader must be recovered"); + String errorMsg = String.format(Locale.ROOT, "%s at least one reader must be recovered", shardId); + logger.error(errorMsg); + throw new IllegalStateException(errorMsg); } boolean success = false; current = null; @@ -120,8 +124,13 @@ public RemoteFsTranslog( } } - public static void download(Repository repository, ShardId shardId, ThreadPool threadPool, Path location) throws IOException { - assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; + public static void download(Repository repository, ShardId shardId, ThreadPool threadPool, Path location, Logger logger) + throws IOException { + assert repository instanceof BlobStoreRepository : String.format( + Locale.ROOT, + "%s repository should be instance of BlobStoreRepository", + shardId + ); BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; FileTransferTracker fileTransferTracker = new FileTransferTracker(shardId); TranslogTransferManager translogTransferManager = buildTranslogTransferManager( @@ -130,11 +139,11 @@ public static void download(Repository repository, ShardId shardId, ThreadPool t shardId, fileTransferTracker ); - RemoteFsTranslog.download(translogTransferManager, location); + RemoteFsTranslog.download(translogTransferManager, location, logger); } - public static void download(TranslogTransferManager translogTransferManager, Path location) throws IOException { - logger.info("Downloading translog files from remote for shard {} ", translogTransferManager.getShardId()); + public static void download(TranslogTransferManager translogTransferManager, Path location, Logger logger) throws IOException { + logger.trace("Downloading translog files from remote"); TranslogTransferMetadata translogMetadata = translogTransferManager.readMetadata(); if (translogMetadata != null) { if (Files.notExists(location)) { @@ -156,7 +165,7 @@ public static void download(TranslogTransferManager translogTransferManager, Pat location.resolve(Translog.CHECKPOINT_FILE_NAME) ); } - logger.info("Downloaded translog files from remote for shard {} ", translogTransferManager.getShardId()); + logger.trace("Downloaded translog files from remote"); } public static TranslogTransferManager buildTranslogTransferManager( @@ -321,8 +330,8 @@ public boolean syncNeeded() { @Override public void close() throws IOException { - assert Translog.calledFromOutsideOrViaTragedyClose() - : "Translog.close method is called from inside Translog, but not via closeOnTragicEvent method"; + assert Translog.calledFromOutsideOrViaTragedyClose() : shardId + + "Translog.close method is called from inside Translog, but not via closeOnTragicEvent method"; if (closed.compareAndSet(false, true)) { try (ReleasableLock lock = writeLock.acquire()) { sync(); @@ -340,12 +349,14 @@ protected long getMinReferencedGen() throws IOException { minGenerationForSeqNo(minSeqNoToKeep, current, readers) ); - assert minReferencedGen >= getMinFileGeneration() : "deletion policy requires a minReferenceGen of [" + assert minReferencedGen >= getMinFileGeneration() : shardId + + " deletion policy requires a minReferenceGen of [" + minReferencedGen + "] but the lowest gen available is [" + getMinFileGeneration() + "]"; - assert minReferencedGen <= currentFileGeneration() : "deletion policy requires a minReferenceGen of [" + assert minReferencedGen <= currentFileGeneration() : shardId + + " deletion policy requires a minReferenceGen of [" + minReferencedGen + "] which is higher than the current generation [" + currentFileGeneration() @@ -356,7 +367,7 @@ protected long getMinReferencedGen() throws IOException { protected void setMinSeqNoToKeep(long seqNo) { if (seqNo < this.minSeqNoToKeep) { throw new IllegalArgumentException( - "min seq number required can't go backwards: " + "current [" + this.minSeqNoToKeep + "] new [" + seqNo + "]" + shardId + " min seq number required can't go backwards: " + "current [" + this.minSeqNoToKeep + "] new [" + seqNo + "]" ); } this.minSeqNoToKeep = seqNo; @@ -416,7 +427,7 @@ private void deleteStaleRemotePrimaryTerms() { // of older primary term. if (olderPrimaryCleaned.trySet(Boolean.TRUE)) { // First we delete all stale primary terms folders from remote store - assert readers.isEmpty() == false : "Expected non-empty readers"; + assert readers.isEmpty() == false : shardId + " Expected non-empty readers"; long minimumReferencedPrimaryTerm = readers.stream().map(BaseTranslogReader::getPrimaryTerm).min(Long::compare).get(); translogTransferManager.deletePrimaryTermsAsync(minimumReferencedPrimaryTerm); } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 6da0ee5521738..54140226e3744 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -8,7 +8,6 @@ package org.opensearch.index.translog.transfer; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.store.IndexInput; @@ -21,6 +20,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.index.shard.ShardId; import org.opensearch.index.translog.Translog; @@ -61,8 +61,7 @@ public class TranslogTransferManager { private static final long TRANSFER_TIMEOUT_IN_MILLIS = 30000; - private static final Logger logger = LogManager.getLogger(TranslogTransferManager.class); - + private final Logger logger; private final static String METADATA_DIR = "metadata"; private final static String DATA_DIR = "data"; @@ -84,6 +83,7 @@ public TranslogTransferManager( this.remoteDataTransferPath = remoteBaseTransferPath.add(DATA_DIR); this.remoteMetadataTransferPath = remoteBaseTransferPath.add(METADATA_DIR); this.fileTransferTracker = fileTransferTracker; + this.logger = Loggers.getLogger(getClass(), shardId); } public ShardId getShardId() { @@ -200,7 +200,7 @@ public TranslogTransferMetadata readMetadata() throws IOException { exceptionSetOnce.set(e); } }, e -> { - logger.error(() -> new ParameterizedMessage("Exception while listing metadata files "), e); + logger.error(() -> new ParameterizedMessage("Exception while listing metadata files"), e); exceptionSetOnce.set((IOException) e); }), latch @@ -295,7 +295,7 @@ public void deleteGenerationAsync(long primaryTerm, Set generations, Runna * @param minPrimaryTermToKeep all primary terms below this primary term are deleted. */ public void deletePrimaryTermsAsync(long minPrimaryTermToKeep) { - logger.info("Deleting primary terms from remote store lesser than {} for {}", minPrimaryTermToKeep, shardId); + logger.info("Deleting primary terms from remote store lesser than {}", minPrimaryTermToKeep); transferService.listFoldersAsync(ThreadPool.Names.REMOTE_PURGE, remoteDataTransferPath, new ActionListener<>() { @Override public void onResponse(Set folders) { @@ -333,7 +333,7 @@ private void deletePrimaryTermAsync(long primaryTerm) { new ActionListener<>() { @Override public void onResponse(Void unused) { - logger.info("Deleted primary term {} for {}", primaryTerm, shardId); + logger.info("Deleted primary term {}", primaryTerm); } @Override @@ -349,12 +349,12 @@ public void delete() { transferService.deleteAsync(ThreadPool.Names.REMOTE_PURGE, remoteBaseTransferPath, new ActionListener<>() { @Override public void onResponse(Void unused) { - logger.info("Deleted all remote translog data for {}", shardId); + logger.info("Deleted all remote translog data"); } @Override public void onFailure(Exception e) { - logger.error("Exception occurred while cleaning translog ", e); + logger.error("Exception occurred while cleaning translog", e); } }); } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 924a9d039da28..5f8aa64457896 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.transfer.FileSnapshot.CheckpointFileSnapshot; @@ -65,6 +66,7 @@ public void setUp() throws Exception { primaryTerm = randomNonNegativeLong(); generation = randomNonNegativeLong(); shardId = mock(ShardId.class); + when(shardId.getIndex()).thenReturn(new Index("index", "indexUUid")); minTranslogGeneration = randomLongBetween(0, generation); remoteBaseTransferPath = new BlobPath().add("base_path"); transferService = mock(TransferService.class); From 7208edc1832c6be3e8548d01289a88370027e5bb Mon Sep 17 00:00:00 2001 From: suraj kumar Date: Tue, 11 Jul 2023 17:11:40 +0530 Subject: [PATCH 099/186] Add wrapped tracer implementation (#8565) * Add wrapped tracer implementation Signed-off-by: suranjay * Add changelog entry Signed-off-by: suranjay * Add @opensearch.internal annotation Signed-off-by: suranjay * Fix test Signed-off-by: suranjay * Fix changelog entry Signed-off-by: suranjay --------- Signed-off-by: suranjay --- CHANGELOG.md | 3 + .../org/opensearch/telemetry/Telemetry.java | 2 + .../telemetry/tracing/AbstractSpan.java | 2 + .../telemetry/tracing/DefaultSpanScope.java | 4 +- .../telemetry/tracing/DefaultTracer.java | 4 +- .../opensearch/telemetry/tracing/Span.java | 4 +- .../telemetry/tracing/SpanReference.java | 4 +- .../tracing/TracerContextStorage.java | 2 + .../tracing/TracingContextPropagator.java | 2 + .../telemetry/tracing/TracingTelemetry.java | 2 + .../telemetry/tracing/noop/NoopSpanScope.java | 2 + .../telemetry/tracing/noop/NoopTracer.java | 2 + .../telemetry/tracing/NoopTracerFactory.java | 2 + ...hreadContextBasedTracerContextStorage.java | 2 + .../telemetry/tracing/TracerFactory.java | 24 +++++-- .../telemetry/tracing/WrappedTracer.java | 52 ++++++++++++++ .../telemetry/tracing/TracerFactoryTests.java | 9 +-- .../telemetry/tracing/WrappedTracerTests.java | 69 +++++++++++++++++++ 18 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java create mode 100644 server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5130dd00845b6..fc367be22c1fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919)) - Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909)) - Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) +- Add wrapper tracer implementation ### Deprecated @@ -164,6 +165,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) - Optimize Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853)) - Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) +- Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) +- Add wrapper tracer implementation ([#8565](https://github.com/opensearch-project/OpenSearch/pull/8565)) ### Deprecated diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java index 6f50699528b6b..65c974a0d0c36 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java @@ -13,6 +13,8 @@ /** * Interface defining telemetry + * + * @opensearch.internal */ public interface Telemetry { diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java index 316edc971913e..150a32b14d0f8 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java @@ -10,6 +10,8 @@ /** * Base span + * + * @opensearch.internal */ public abstract class AbstractSpan implements Span { diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java index 58e9e0abad739..356b72187de74 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java @@ -12,8 +12,10 @@ /** * Default implementation of Scope + * + * @opensearch.internal */ -public class DefaultSpanScope implements SpanScope { +final class DefaultSpanScope implements SpanScope { private final Span span; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java index 783edd238c1c2..ea59eec645420 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java @@ -16,9 +16,9 @@ * The default tracer implementation. It handles tracing context propagation between spans by maintaining * current active span in its storage * - * + * @opensearch.internal */ -public class DefaultTracer implements Tracer { +class DefaultTracer implements Tracer { static final String THREAD_NAME = "th_name"; private final TracingTelemetry tracingTelemetry; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java index d60b4e60adece..6cb1c8234f3de 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Span.java @@ -11,7 +11,9 @@ /** * An interface that represents a tracing span. * Spans are created by the Tracer.startSpan method. - * Span must be ended by calling Tracer.endSpan which internally calls Span's endSpan. + * Span must be ended by calling SpanScope.close which internally calls Span's endSpan. + * + * @opensearch.internal */ public interface Span { diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java index 99d1bd3c93c84..180136ecf7a57 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanReference.java @@ -10,8 +10,10 @@ /** * Wrapper class to hold reference of Span + * + * @opensearch.internal */ -public class SpanReference { +final class SpanReference { private Span span; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java index eb93006835332..d85b404b0ce41 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracerContextStorage.java @@ -12,6 +12,8 @@ * Storage interface used for storing tracing context * @param key type * @param value type + * + * @opensearch.internal */ public interface TracerContextStorage { /** diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java index 1152e3aedfa88..3e4a377d33a3d 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingContextPropagator.java @@ -13,6 +13,8 @@ /** * Interface defining the tracing related context propagation + * + * @opensearch.internal */ public interface TracingContextPropagator { diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java index 16c76bd0cc141..bce955fc2d99e 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java @@ -12,6 +12,8 @@ /** * Interface for tracing telemetry providers + * + * @opensearch.internal */ public interface TracingTelemetry extends Closeable { diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java index c0dbaf65ba48b..a1d16d1d80d00 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopSpanScope.java @@ -12,6 +12,8 @@ /** * No-op implementation of SpanScope + * + * @opensearch.internal */ public final class NoopSpanScope implements SpanScope { diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java index a66cbcf4fef52..a1768d7d59116 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java @@ -13,6 +13,8 @@ /** * No-op implementation of Tracer + * + * @opensearch.internal */ public class NoopTracer implements Tracer { diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java b/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java index 3d7f8133788ce..f82a390dc1754 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/NoopTracerFactory.java @@ -14,6 +14,8 @@ /** * No-op implementation of TracerFactory + * + * @opensearch.internal */ public class NoopTracerFactory extends TracerFactory { public NoopTracerFactory() { diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java b/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java index 0d0b795fdc715..c009ab2391aab 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java @@ -18,6 +18,8 @@ /** * Core's ThreadContext based TracerContextStorage implementation + * + * @opensearch.internal */ public class ThreadContextBasedTracerContextStorage implements TracerContextStorage, ThreadContextStatePropagator { diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java b/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java index 8228cded4c822..d8fe812c82f53 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/TracerFactory.java @@ -21,7 +21,7 @@ /** * TracerManager represents a single global class that is used to access tracers. - * + *

* The Tracer singleton object can be retrieved using tracerManager.getTracer(). The TracerManager object * is created during class initialization and cannot subsequently be changed. */ @@ -30,21 +30,20 @@ public class TracerFactory implements Closeable { private static final Logger logger = LogManager.getLogger(TracerFactory.class); private final TelemetrySettings telemetrySettings; - private final Tracer defaultTracer; + private final Tracer tracer; public TracerFactory(TelemetrySettings telemetrySettings, Optional telemetry, ThreadContext threadContext) { this.telemetrySettings = telemetrySettings; - this.defaultTracer = telemetry.map(Telemetry::getTracingTelemetry) - .map(tracingTelemetry -> createDefaultTracer(tracingTelemetry, threadContext)) - .orElse(NoopTracer.INSTANCE); + this.tracer = tracer(telemetry, threadContext); } /** * Returns the tracer instance + * * @return tracer instance */ public Tracer getTracer() { - return telemetrySettings.isTracingEnabled() ? defaultTracer : NoopTracer.INSTANCE; + return tracer; } /** @@ -53,12 +52,19 @@ public Tracer getTracer() { @Override public void close() { try { - defaultTracer.close(); + tracer.close(); } catch (IOException e) { logger.warn("Error closing tracer", e); } } + private Tracer tracer(Optional telemetry, ThreadContext threadContext) { + return telemetry.map(Telemetry::getTracingTelemetry) + .map(tracingTelemetry -> createDefaultTracer(tracingTelemetry, threadContext)) + .map(defaultTracer -> createWrappedTracer(defaultTracer)) + .orElse(NoopTracer.INSTANCE); + } + private Tracer createDefaultTracer(TracingTelemetry tracingTelemetry, ThreadContext threadContext) { TracerContextStorage tracerContextStorage = new ThreadContextBasedTracerContextStorage( threadContext, @@ -67,4 +73,8 @@ private Tracer createDefaultTracer(TracingTelemetry tracingTelemetry, ThreadCont return new DefaultTracer(tracingTelemetry, tracerContextStorage); } + private Tracer createWrappedTracer(Tracer defaultTracer) { + return new WrappedTracer(telemetrySettings, defaultTracer); + } + } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java new file mode 100644 index 0000000000000..0ba9a8ea5fd88 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; + +import java.io.IOException; + +/** + * Wrapper implementation of Tracer. This delegates call to right tracer based on the tracer settings + * + * @opensearch.internal + */ +final class WrappedTracer implements Tracer { + + private final Tracer defaultTracer; + private final TelemetrySettings telemetrySettings; + + /** + * Creates WrappedTracer instance + * + * @param telemetrySettings telemetry settings + * @param defaultTracer default tracer instance + */ + public WrappedTracer(TelemetrySettings telemetrySettings, Tracer defaultTracer) { + this.defaultTracer = defaultTracer; + this.telemetrySettings = telemetrySettings; + } + + @Override + public SpanScope startSpan(String spanName) { + Tracer delegateTracer = getDelegateTracer(); + return delegateTracer.startSpan(spanName); + } + + @Override + public void close() throws IOException { + defaultTracer.close(); + } + + // visible for testing + Tracer getDelegateTracer() { + return telemetrySettings.isTracingEnabled() ? defaultTracer : NoopTracer.INSTANCE; + } +} diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java index df9cdd6669d23..0ffccee505d43 100644 --- a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java +++ b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java @@ -36,19 +36,20 @@ public void close() { tracerFactory.close(); } - public void testGetTracerWithTracingDisabledReturnsNoopTracer() { + public void testGetTracerWithUnavailableTracingTelemetryReturnsNoopTracer() { Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), false).build(); TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); Telemetry mockTelemetry = mock(Telemetry.class); when(mockTelemetry.getTracingTelemetry()).thenReturn(mock(TracingTelemetry.class)); - tracerFactory = new TracerFactory(telemetrySettings, Optional.of(mockTelemetry), new ThreadContext(Settings.EMPTY)); + tracerFactory = new TracerFactory(telemetrySettings, Optional.empty(), new ThreadContext(Settings.EMPTY)); Tracer tracer = tracerFactory.getTracer(); + assertTrue(tracer instanceof NoopTracer); assertTrue(tracer.startSpan("foo") == SpanScope.NO_OP); } - public void testGetTracerWithTracingEnabledReturnsDefaultTracer() { + public void testGetTracerWithAvailableTracingTelemetryReturnsWrappedTracer() { Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true).build(); TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); Telemetry mockTelemetry = mock(Telemetry.class); @@ -56,7 +57,7 @@ public void testGetTracerWithTracingEnabledReturnsDefaultTracer() { tracerFactory = new TracerFactory(telemetrySettings, Optional.of(mockTelemetry), new ThreadContext(Settings.EMPTY)); Tracer tracer = tracerFactory.getTracer(); - assertTrue(tracer instanceof DefaultTracer); + assertTrue(tracer instanceof WrappedTracer); } diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java new file mode 100644 index 0000000000000..d1abc5a4d98aa --- /dev/null +++ b/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class WrappedTracerTests extends OpenSearchTestCase { + + public void testStartSpanWithTracingDisabledInvokesNoopTracer() throws Exception { + Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), false).build(); + TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); + DefaultTracer mockDefaultTracer = mock(DefaultTracer.class); + + try (WrappedTracer wrappedTracer = new WrappedTracer(telemetrySettings, mockDefaultTracer)) { + wrappedTracer.startSpan("foo"); + assertTrue(wrappedTracer.getDelegateTracer() instanceof NoopTracer); + verify(mockDefaultTracer, never()).startSpan("foo"); + } + } + + public void testStartSpanWithTracingEnabledInvokesDefaultTracer() throws Exception { + Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true).build(); + TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); + DefaultTracer mockDefaultTracer = mock(DefaultTracer.class); + + try (WrappedTracer wrappedTracer = new WrappedTracer(telemetrySettings, mockDefaultTracer)) { + wrappedTracer.startSpan("foo"); + + assertTrue(wrappedTracer.getDelegateTracer() instanceof DefaultTracer); + verify(mockDefaultTracer).startSpan("foo"); + } + } + + public void testClose() throws IOException { + DefaultTracer mockDefaultTracer = mock(DefaultTracer.class); + WrappedTracer wrappedTracer = new WrappedTracer(null, mockDefaultTracer); + + wrappedTracer.close(); + + verify(mockDefaultTracer).close(); + } + + private Set> getClusterSettings() { + Set> allTracerSettings = new HashSet<>(); + ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add)); + return allTracerSettings; + } +} From 5f98d674c74ab5d24195993734bb054840906b47 Mon Sep 17 00:00:00 2001 From: Raghuvansh Raj Date: Tue, 11 Jul 2023 18:06:01 +0530 Subject: [PATCH 100/186] [Remote Store] Add multipart upload integration for translog and segment files (#7119) Signed-off-by: Raghuvansh Raj --- .../common/io/InputStreamContainer.java | 11 +- .../RemoteStoreBaseIntegTestCase.java | 4 +- .../RemoteStoreMultipartFileCorruptionIT.java | 111 +++++++++ .../multipart/RemoteStoreMultipartIT.java | 38 +++ .../multipart/mocks/MockFsBlobStore.java | 36 +++ .../multipart/mocks/MockFsRepository.java | 46 ++++ .../mocks/MockFsRepositoryPlugin.java | 38 +++ .../mocks/MockFsVerifyingBlobContainer.java | 120 +++++++++ .../VerifyingMultiStreamBlobContainer.java | 34 +++ .../transfer/RemoteTransferContainer.java | 2 +- .../org/opensearch/common/util/ByteUtils.java | 10 + .../shard/RemoteStoreRefreshListener.java | 227 ++++++++---------- .../index/store/RemoteDirectory.java | 4 + .../store/RemoteSegmentStoreDirectory.java | 127 ++++++++++ .../ChecksumCombinationException.java | 2 +- .../opensearch/index/translog/Checkpoint.java | 2 +- .../TranslogCheckedContainer.java | 17 +- .../index/translog/TranslogHeader.java | 12 +- .../index/translog/TranslogReader.java | 37 ++- .../index/translog/TranslogWriter.java | 25 +- .../index/translog/checked/package-info.java | 10 - .../transfer/BlobStoreTransferService.java | 98 +++++++- .../index/translog/transfer/FileSnapshot.java | 17 +- .../translog/transfer/TransferService.java | 28 ++- .../TranslogCheckpointTransferSnapshot.java | 10 +- .../transfer/TranslogTransferManager.java | 15 +- .../org/opensearch/threadpool/ThreadPool.java | 2 +- .../RemoteTransferContainerTests.java | 41 ++++ .../RemoteSegmentStoreDirectoryTests.java | 86 +++++++ .../index/store/TestUploadListener.java | 43 ++++ ...oreTransferServiceMockRepositoryTests.java | 189 +++++++++++++++ .../BlobStoreTransferServiceTests.java | 22 +- .../translog/transfer/FileSnapshotTests.java | 6 +- .../transfer/FileTransferTrackerTests.java | 12 +- .../TranslogTransferManagerTests.java | 31 ++- 35 files changed, 1293 insertions(+), 220 deletions(-) rename {server => libs/common}/src/main/java/org/opensearch/common/io/InputStreamContainer.java (85%) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepository.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepositoryPlugin.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java create mode 100644 server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java rename server/src/main/java/org/opensearch/index/translog/{checked => }/TranslogCheckedContainer.java (74%) delete mode 100644 server/src/main/java/org/opensearch/index/translog/checked/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/store/TestUploadListener.java create mode 100644 server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java diff --git a/server/src/main/java/org/opensearch/common/io/InputStreamContainer.java b/libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java similarity index 85% rename from server/src/main/java/org/opensearch/common/io/InputStreamContainer.java rename to libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java index ce5dcff9f5349..eb8a4e1382497 100644 --- a/server/src/main/java/org/opensearch/common/io/InputStreamContainer.java +++ b/libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java @@ -19,6 +19,7 @@ public class InputStreamContainer { private final InputStream inputStream; private final long contentLength; + private final long offset; /** * Construct a new stream object @@ -26,9 +27,10 @@ public class InputStreamContainer { * @param inputStream The input stream that is to be encapsulated * @param contentLength The total content length that is to be read from the stream */ - public InputStreamContainer(InputStream inputStream, long contentLength) { + public InputStreamContainer(InputStream inputStream, long contentLength, long offset) { this.inputStream = inputStream; this.contentLength = contentLength; + this.offset = offset; } /** @@ -44,4 +46,11 @@ public InputStream getInputStream() { public long getContentLength() { return contentLength; } + + /** + * @return offset of the source content. + */ + public long getOffset() { + return offset; + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 2b3fcadfc645e..10f01749ab4c5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -102,9 +102,7 @@ protected void putRepository(Path path) { protected void setupRepo() { internalCluster().startClusterManagerOnlyNode(); absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); + putRepository(absolutePath); } @After diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java new file mode 100644 index 0000000000000..8f375ca6e2b01 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore.multipart; + +import org.junit.After; +import org.junit.Before; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.IndexModule; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotestore.multipart.mocks.MockFsRepository; +import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class RemoteStoreMultipartFileCorruptionIT extends OpenSearchIntegTestCase { + + protected static final String REPOSITORY_NAME = "test-remore-store-repo"; + private static final String INDEX_NAME = "remote-store-test-idx-1"; + + @Override + protected Collection> nodePlugins() { + return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); + } + + @Before + public void setup() { + internalCluster().startClusterManagerOnlyNode(); + Path absolutePath = randomRepoPath().toAbsolutePath(); + putRepository(absolutePath); + } + + protected void putRepository(Path path) { + assertAcked( + clusterAdmin().preparePutRepository(REPOSITORY_NAME) + .setType(MockFsRepositoryPlugin.TYPE) + .setSettings( + Settings.builder() + .put("location", path) + // custom setting for MockFsRepositoryPlugin + .put(MockFsRepository.TRIGGER_DATA_INTEGRITY_FAILURE.getKey(), true) + ) + ); + } + + @After + public void teardown() { + assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + } + + protected Settings remoteStoreIndexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put("index.refresh_interval", "300s") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) + .build(); + } + + private IndexResponse indexSingleDoc() { + return client().prepareIndex(INDEX_NAME) + .setId(UUIDs.randomBase64UUID()) + .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) + .get(); + } + + public void testLocalFileCorruptionDuringUpload() { + internalCluster().startDataOnlyNodes(1); + createIndex(INDEX_NAME, remoteStoreIndexSettings()); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + indexSingleDoc(); + + client().admin() + .indices() + .prepareRefresh(INDEX_NAME) + .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED) + .execute() + .actionGet(); + + // ensuring red cluster meaning shard has failed and is unassigned + ensureRed(INDEX_NAME); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java new file mode 100644 index 0000000000000..a523d5c0f5470 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore.multipart; + +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotestore.RemoteStoreIT; +import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class RemoteStoreMultipartIT extends RemoteStoreIT { + + @Override + protected Collection> nodePlugins() { + return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); + } + + @Override + protected void putRepository(Path path) { + assertAcked( + clusterAdmin().preparePutRepository(REPOSITORY_NAME) + .setType(MockFsRepositoryPlugin.TYPE) + .setSettings(Settings.builder().put("location", path)) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java new file mode 100644 index 0000000000000..f1d9fbba84528 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsBlobStore.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore.multipart.mocks; + +import org.opensearch.OpenSearchException; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.fs.FsBlobStore; + +import java.io.IOException; +import java.nio.file.Path; + +public class MockFsBlobStore extends FsBlobStore { + + private final boolean triggerDataIntegrityFailure; + + public MockFsBlobStore(int bufferSizeInBytes, Path path, boolean readonly, boolean triggerDataIntegrityFailure) throws IOException { + super(bufferSizeInBytes, path, readonly); + this.triggerDataIntegrityFailure = triggerDataIntegrityFailure; + } + + @Override + public BlobContainer blobContainer(BlobPath path) { + try { + return new MockFsVerifyingBlobContainer(this, path, buildAndCreate(path), triggerDataIntegrityFailure); + } catch (IOException ex) { + throw new OpenSearchException("failed to create blob container", ex); + } + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepository.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepository.java new file mode 100644 index 0000000000000..15a9853477081 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepository.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore.multipart.mocks; + +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.fs.FsBlobStore; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.repositories.fs.FsRepository; + +public class MockFsRepository extends FsRepository { + + public static Setting TRIGGER_DATA_INTEGRITY_FAILURE = Setting.boolSetting( + "mock_fs_repository.trigger_data_integrity_failure", + false + ); + + private final boolean triggerDataIntegrityFailure; + + public MockFsRepository( + RepositoryMetadata metadata, + Environment environment, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + RecoverySettings recoverySettings + ) { + super(metadata, environment, namedXContentRegistry, clusterService, recoverySettings); + triggerDataIntegrityFailure = TRIGGER_DATA_INTEGRITY_FAILURE.get(metadata.settings()); + } + + @Override + protected BlobStore createBlobStore() throws Exception { + FsBlobStore fsBlobStore = (FsBlobStore) super.createBlobStore(); + return new MockFsBlobStore(fsBlobStore.bufferSizeInBytes(), fsBlobStore.path(), isReadOnly(), triggerDataIntegrityFailure); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepositoryPlugin.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepositoryPlugin.java new file mode 100644 index 0000000000000..ffd53adf4e29e --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsRepositoryPlugin.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore.multipart.mocks; + +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.RepositoryPlugin; +import org.opensearch.repositories.Repository; + +import java.util.Collections; +import java.util.Map; + +public class MockFsRepositoryPlugin extends Plugin implements RepositoryPlugin { + + public static final String TYPE = "fs_multipart_repository"; + + @Override + public Map getRepositories( + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + RecoverySettings recoverySettings + ) { + return Collections.singletonMap( + "fs_multipart_repository", + metadata -> new MockFsRepository(metadata, env, namedXContentRegistry, clusterService, recoverySettings) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java new file mode 100644 index 0000000000000..8f2814eb7c4c4 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsVerifyingBlobContainer.java @@ -0,0 +1,120 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore.multipart.mocks; + +import org.apache.lucene.index.CorruptIndexException; +import org.opensearch.action.ActionListener; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.fs.FsBlobContainer; +import org.opensearch.common.blobstore.fs.FsBlobStore; +import org.opensearch.common.blobstore.stream.write.WriteContext; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class MockFsVerifyingBlobContainer extends FsBlobContainer implements VerifyingMultiStreamBlobContainer { + + private static final int TRANSFER_TIMEOUT_MILLIS = 30000; + + private final boolean triggerDataIntegrityFailure; + + public MockFsVerifyingBlobContainer(FsBlobStore blobStore, BlobPath blobPath, Path path, boolean triggerDataIntegrityFailure) { + super(blobStore, blobPath, path); + this.triggerDataIntegrityFailure = triggerDataIntegrityFailure; + } + + @Override + public void asyncBlobUpload(WriteContext writeContext, ActionListener completionListener) throws IOException { + + int nParts = 10; + long partSize = writeContext.getFileSize() / nParts; + StreamContext streamContext = writeContext.getStreamProvider(partSize); + final Path file = path.resolve(writeContext.getFileName()); + byte[] buffer = new byte[(int) writeContext.getFileSize()]; + AtomicLong totalContentRead = new AtomicLong(); + CountDownLatch latch = new CountDownLatch(streamContext.getNumberOfParts()); + for (int partIdx = 0; partIdx < streamContext.getNumberOfParts(); partIdx++) { + int finalPartIdx = partIdx; + Thread thread = new Thread(() -> { + try { + InputStreamContainer inputStreamContainer = streamContext.provideStream(finalPartIdx); + InputStream inputStream = inputStreamContainer.getInputStream(); + long remainingContentLength = inputStreamContainer.getContentLength(); + long offset = partSize * finalPartIdx; + while (remainingContentLength > 0) { + int readContentLength = inputStream.read(buffer, (int) offset, (int) remainingContentLength); + totalContentRead.addAndGet(readContentLength); + remainingContentLength -= readContentLength; + offset += readContentLength; + } + inputStream.close(); + } catch (IOException e) { + completionListener.onFailure(e); + } finally { + latch.countDown(); + } + }); + thread.start(); + } + try { + if (!latch.await(TRANSFER_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IOException("Timed out waiting for file transfer to complete for " + writeContext.getFileName()); + } + } catch (InterruptedException e) { + throw new IOException("Await interrupted on CountDownLatch, transfer failed for " + writeContext.getFileName()); + } + try (OutputStream outputStream = Files.newOutputStream(file, StandardOpenOption.CREATE_NEW)) { + outputStream.write(buffer); + } + if (writeContext.getFileSize() != totalContentRead.get()) { + throw new IOException( + "Incorrect content length read for file " + + writeContext.getFileName() + + ", actual file size: " + + writeContext.getFileSize() + + ", bytes read: " + + totalContentRead.get() + ); + } + + try { + // bulks need to succeed for segment files to be generated + if (isSegmentFile(writeContext.getFileName()) && triggerDataIntegrityFailure) { + completionListener.onFailure( + new RuntimeException( + new CorruptIndexException( + "Data integrity check failure for file: " + writeContext.getFileName(), + writeContext.getFileName() + ) + ) + ); + } else { + writeContext.getUploadFinalizer().accept(true); + completionListener.onResponse(null); + } + } catch (Exception e) { + completionListener.onFailure(e); + } + + } + + private boolean isSegmentFile(String filename) { + return !filename.endsWith(".tlog") && !filename.endsWith(".ckp"); + } +} diff --git a/server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java new file mode 100644 index 0000000000000..0dfcc5c50e4b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/blobstore/VerifyingMultiStreamBlobContainer.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.blobstore; + +import org.opensearch.action.ActionListener; +import org.opensearch.common.blobstore.stream.write.WriteContext; + +import java.io.IOException; + +/** + * An extension of {@link BlobContainer} that adds {@link VerifyingMultiStreamBlobContainer#asyncBlobUpload} to allow + * multipart uploads and performs integrity checks on transferred files + * + * @opensearch.internal + */ +public interface VerifyingMultiStreamBlobContainer extends BlobContainer { + + /** + * Reads blob content from multiple streams, each from a specific part of the file, which is provided by the + * StreamContextSupplier in the WriteContext passed to this method. An {@link IOException} is thrown if reading + * any of the input streams fails, or writing to the target blob fails + * + * @param writeContext A WriteContext object encapsulating all information needed to perform the upload + * @param completionListener Listener on which upload events should be published. + * @throws IOException if any of the input streams could not be read, or the target blob could not be written to + */ + void asyncBlobUpload(WriteContext writeContext, ActionListener completionListener) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java b/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java index 7864c3ab5c794..ca744efae902d 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java @@ -160,7 +160,7 @@ private LocalStreamSupplier getMultipartStreamSupplier( : offsetRangeInputStream; Objects.requireNonNull(inputStreams.get())[streamIdx] = inputStream; - return new InputStreamContainer(inputStream, size); + return new InputStreamContainer(inputStream, size, position); } catch (IOException e) { log.error("Failed to create input stream", e); throw e; diff --git a/server/src/main/java/org/opensearch/common/util/ByteUtils.java b/server/src/main/java/org/opensearch/common/util/ByteUtils.java index 36ae3b1f5bcaa..8c7665d991751 100644 --- a/server/src/main/java/org/opensearch/common/util/ByteUtils.java +++ b/server/src/main/java/org/opensearch/common/util/ByteUtils.java @@ -61,6 +61,16 @@ public static void writeLongLE(long l, byte[] arr, int offset) { assert l == 0; } + /** Convert long to a byte array in big-endian format */ + public static byte[] toByteArrayBE(long l) { + byte[] result = new byte[8]; + for (int i = 7; i >= 0; i--) { + result[i] = (byte) (l & 0xffL); + l >>= 8; + } + return result; + } + /** Write a long in little-endian format. */ public static long readLongLE(byte[] arr, int offset) { long l = arr[offset++] & 0xFFL; diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 46d52bc8ca5df..e087bbb265727 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.search.ReferenceManager; @@ -18,11 +19,14 @@ import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; import org.opensearch.action.bulk.BackoffPolicy; -import org.opensearch.common.CheckedFunction; +import org.opensearch.action.support.GroupedActionListener; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.logging.Loggers; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.UploadListener; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.engine.EngineException; import org.opensearch.index.engine.InternalEngine; @@ -46,6 +50,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -110,7 +115,7 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres private final SegmentReplicationCheckpointPublisher checkpointPublisher; - private final FileUploader fileUploader; + private final UploadListener statsListener; public RemoteStoreRefreshListener( IndexShard indexShard, @@ -137,7 +142,7 @@ public RemoteStoreRefreshListener( this.segmentTracker = segmentTracker; resetBackOffDelayIterator(); this.checkpointPublisher = checkpointPublisher; - this.fileUploader = new FileUploader(new UploadTracker() { + this.statsListener = new UploadListener() { @Override public void beforeUpload(String file) { // Start tracking the upload bytes started @@ -156,7 +161,7 @@ public void onFailure(String file) { // Track upload failure segmentTracker.addUploadBytesFailed(latestFileNameSizeOnLocalMap.get(file)); } - }, remoteDirectory, storeDirectory, this::getChecksumOfLocalFile, logger); + }; } @Override @@ -190,7 +195,7 @@ private synchronized void syncSegments(boolean isRetry) { long refreshTimeMs = segmentTracker.getLocalRefreshTimeMs(), refreshClockTimeMs = segmentTracker.getLocalRefreshClockTimeMs(); long refreshSeqNo = segmentTracker.getLocalRefreshSeqNo(); long bytesBeforeUpload = segmentTracker.getUploadBytesSucceeded(), startTimeInNS = System.nanoTime(); - boolean shouldRetry = true; + try { if (this.primaryTerm != indexShard.getOperationPrimaryTerm()) { @@ -242,18 +247,51 @@ private synchronized void syncSegments(boolean isRetry) { // Create a map of file name to size and update the refresh segment tracker updateLocalSizeMapAndTracker(localSegmentsPostRefresh); + CountDownLatch latch = new CountDownLatch(1); + ActionListener segmentUploadsCompletedListener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + boolean shouldRetry = true; + try { + // Start metadata file upload + uploadMetadata(localSegmentsPostRefresh, segmentInfos); + clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); + onSuccessfulSegmentsSync( + refreshTimeMs, + refreshClockTimeMs, + refreshSeqNo, + lastRefreshedCheckpoint, + checkpoint + ); + // At this point since we have uploaded new segments, segment infos and segment metadata file, + // along with marking minSeqNoToKeep, upload has succeeded completely. + shouldRetry = false; + } catch (Exception e) { + // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried + // in the next refresh. This should not affect durability of the indexed data after remote trans-log + // integration. + logger.warn("Exception in post new segment upload actions", e); + } finally { + doComplete(shouldRetry); + } + } + + @Override + public void onFailure(Exception e) { + logger.warn("Exception while uploading new segments to the remote segment store", e); + doComplete(true); + } + + private void doComplete(boolean shouldRetry) { + // Update the segment tracker with the final upload status as seen at the end + updateFinalUploadStatusInSegmentTracker(shouldRetry == false, bytesBeforeUpload, startTimeInNS); + afterSegmentsSync(isRetry, shouldRetry); + } + }, latch); // Start the segments files upload - boolean newSegmentsUploadStatus = uploadNewSegments(localSegmentsPostRefresh); - if (newSegmentsUploadStatus) { - // Start metadata file upload - uploadMetadata(localSegmentsPostRefresh, segmentInfos); - clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); - onSuccessfulSegmentsSync(refreshTimeMs, refreshClockTimeMs, refreshSeqNo, lastRefreshedCheckpoint, checkpoint); - // At this point since we have uploaded new segments, segment infos and segment metadata file, - // along with marking minSeqNoToKeep, upload has succeeded completely. - shouldRetry = false; - } + uploadNewSegments(localSegmentsPostRefresh, segmentUploadsCompletedListener); + latch.await(); } } catch (EngineException e) { logger.warn("Exception while reading SegmentInfosSnapshot", e); @@ -265,11 +303,7 @@ private synchronized void syncSegments(boolean isRetry) { } } catch (Throwable t) { logger.error("Exception in RemoteStoreRefreshListener.afterRefresh()", t); - } finally { - // Update the segment tracker with the final upload status as seen at the end - updateFinalUploadStatusInSegmentTracker(shouldRetry == false, bytesBeforeUpload, startTimeInNS); } - afterSegmentsSync(isRetry, shouldRetry); } /** @@ -378,17 +412,50 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se } } - private boolean uploadNewSegments(Collection localSegmentsPostRefresh) throws IOException { - AtomicBoolean uploadSuccess = new AtomicBoolean(true); - localSegmentsPostRefresh.forEach(file -> { - try { - fileUploader.uploadFile(file); - } catch (IOException e) { - uploadSuccess.set(false); - logger.warn(() -> new ParameterizedMessage("Exception while uploading file {} to the remote segment store", file), e); - } - }); - return uploadSuccess.get(); + private void uploadNewSegments(Collection localSegmentsPostRefresh, ActionListener listener) { + Collection filteredFiles = localSegmentsPostRefresh.stream().filter(file -> !skipUpload(file)).collect(Collectors.toList()); + if (filteredFiles.size() == 0) { + listener.onResponse(null); + return; + } + + ActionListener> mappedListener = ActionListener.map(listener, resp -> null); + GroupedActionListener batchUploadListener = new GroupedActionListener<>(mappedListener, filteredFiles.size()); + + for (String src : filteredFiles) { + ActionListener aggregatedListener = ActionListener.wrap(resp -> { + statsListener.onSuccess(src); + batchUploadListener.onResponse(resp); + }, ex -> { + logger.warn(() -> new ParameterizedMessage("Exception: [{}] while uploading segment files", ex), ex); + if (ex instanceof CorruptIndexException) { + indexShard.failShard(ex.getMessage(), ex); + } + statsListener.onFailure(src); + batchUploadListener.onFailure(ex); + }); + statsListener.beforeUpload(src); + remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener); + } + } + + /** + * Whether to upload a file or not depending on whether file is in excluded list or has been already uploaded. + * + * @param file that needs to be uploaded. + * @return true if the upload has to be skipped for the file. + */ + private boolean skipUpload(String file) { + try { + // Exclude files that are already uploaded and the exclude files to come up with the list of files to be uploaded. + return EXCLUDE_FILES.contains(file) || remoteDirectory.containsFile(file, getChecksumOfLocalFile(file)); + } catch (IOException e) { + logger.error( + "Exception while reading checksum of local segment file: {}, ignoring the exception and re-uploading the file", + file + ); + } + return false; } private String getChecksumOfLocalFile(String file) throws IOException { @@ -462,104 +529,4 @@ private void updateFinalUploadStatusInSegmentTracker(boolean uploadStatus, long segmentTracker.incrementTotalUploadsFailed(); } } - - /** - * This class is a wrapper over the copying of file from local to remote store allowing to decorate the actual copy - * method along with adding hooks of code that can be run before, on success and on failure. - * - * @opensearch.internal - */ - private static class FileUploader { - - private final Logger logger; - - private final UploadTracker uploadTracker; - - private final RemoteSegmentStoreDirectory remoteDirectory; - - private final Directory storeDirectory; - - private final CheckedFunction checksumProvider; - - public FileUploader( - UploadTracker uploadTracker, - RemoteSegmentStoreDirectory remoteDirectory, - Directory storeDirectory, - CheckedFunction checksumProvider, - Logger logger - ) { - this.uploadTracker = uploadTracker; - this.remoteDirectory = remoteDirectory; - this.storeDirectory = storeDirectory; - this.checksumProvider = checksumProvider; - this.logger = logger; - } - - /** - * Calling this method will lead to before getting executed and then the actual upload. Based on the upload status, - * the onSuccess or onFailure method gets invoked. - * - * @param file the file which is to be uploaded. - * @throws IOException is thrown if the upload fails. - */ - private void uploadFile(String file) throws IOException { - if (skipUpload(file)) { - return; - } - uploadTracker.beforeUpload(file); - boolean success = false; - try { - performUpload(file); - uploadTracker.onSuccess(file); - success = true; - } finally { - if (!success) { - uploadTracker.onFailure(file); - } - } - } - - /** - * Whether to upload a file or not depending on whether file is in excluded list or has been already uploaded. - * - * @param file that needs to be uploaded. - * @return true if the upload has to be skipped for the file. - */ - private boolean skipUpload(String file) { - try { - // Exclude files that are already uploaded and the exclude files to come up with the list of files to be uploaded. - return EXCLUDE_FILES.contains(file) || remoteDirectory.containsFile(file, checksumProvider.apply(file)); - } catch (IOException e) { - logger.error( - "Exception while reading checksum of local segment file: {}, ignoring the exception and re-uploading the file", - file - ); - } - return false; - } - - /** - * This method does the actual upload. - * - * @param file that needs to be uploaded. - * @throws IOException is thrown if the upload fails. - */ - private void performUpload(String file) throws IOException { - remoteDirectory.copyFrom(storeDirectory, file, file, IOContext.DEFAULT); - } - } - - /** - * A tracker class that is fed to FileUploader. - * - * @opensearch.internal - */ - interface UploadTracker { - - void beforeUpload(String file); - - void onSuccess(String file); - - void onFailure(String file); - } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 8782808c070ab..f7fe7ca62e6ba 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -45,6 +45,10 @@ public class RemoteDirectory extends Directory { protected final BlobContainer blobContainer; + public BlobContainer getBlobContainer() { + return blobContainer; + } + public RemoteDirectory(BlobContainer blobContainer) { this.blobContainer = blobContainer; } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index e7602203440d2..395ecba442e86 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -8,9 +8,12 @@ package org.opensearch.index.store; +import com.jcraft.jzlib.JZlib; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.store.ByteBuffersIndexOutput; @@ -19,10 +22,20 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.opensearch.ExceptionsHelper; +import org.opensearch.action.ActionListener; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.exception.CorruptFileException; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.common.util.ByteUtils; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.store.exception.ChecksumCombinationException; import org.opensearch.index.store.lockmanager.FileLockInfo; import org.opensearch.index.store.lockmanager.RemoteStoreCommitLevelLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; @@ -44,6 +57,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import java.util.zip.CRC32; /** * A RemoteDirectory extension for remote segment store. We need to make sure we don't overwrite a segment file once uploaded. @@ -62,6 +76,11 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement */ public static final String SEGMENT_NAME_UUID_SEPARATOR = "__"; + /** + * Number of bytes in the segment file to store checksum + */ + private static final int SEGMENT_CHECKSUM_BYTES = 8; + /** * remoteDataDirectory is used to store segment files at path: cluster_UUID/index_UUID/shardId/segments/data */ @@ -349,6 +368,89 @@ public IndexInput openInput(String name, IOContext context) throws IOException { } } + /** + * Copies a file from the source directory to a remote based on multi-stream upload support. + * If vendor plugin supports uploading multiple parts in parallel, BlobContainer#writeBlobByStreams + * will be used, else, the legacy {@link RemoteSegmentStoreDirectory#copyFrom(Directory, String, String, IOContext)} + * will be called. + * + * @param from The directory for the file to be uploaded + * @param src File to be uploaded + * @param context IOContext to be used to open IndexInput of file during remote upload + * @param listener Listener to handle upload callback events + */ + public void copyFrom(Directory from, String src, IOContext context, ActionListener listener) { + if (remoteDataDirectory.getBlobContainer() instanceof VerifyingMultiStreamBlobContainer) { + try { + String remoteFilename = getNewRemoteSegmentFilename(src); + uploadBlob(from, src, remoteFilename, context, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } else { + try { + copyFrom(from, src, src, context); + listener.onResponse(null); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("Exception while uploading file {} to the remote segment store", src), e); + listener.onFailure(e); + } + } + } + + private void uploadBlob(Directory from, String src, String remoteFileName, IOContext ioContext, ActionListener listener) + throws Exception { + long expectedChecksum = calculateChecksumOfChecksum(from, src); + long contentLength; + try (IndexInput indexInput = from.openInput(src, ioContext)) { + contentLength = indexInput.length(); + } + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + src, + remoteFileName, + contentLength, + true, + WritePriority.NORMAL, + (size, position) -> new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position), + expectedChecksum, + remoteDataDirectory.getBlobContainer() instanceof VerifyingMultiStreamBlobContainer + ); + ActionListener completionListener = ActionListener.wrap(resp -> { + try { + postUpload(from, src, remoteFileName, getChecksumOfLocalFile(from, src)); + listener.onResponse(null); + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("Exception in segment postUpload for file [{}]", src), e); + listener.onFailure(e); + } + }, ex -> { + logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", src), ex); + IOException corruptIndexException = ExceptionsHelper.unwrapCorruption(ex); + if (corruptIndexException != null) { + listener.onFailure(corruptIndexException); + return; + } + Throwable throwable = ExceptionsHelper.unwrap(ex, CorruptFileException.class); + if (throwable != null) { + CorruptFileException corruptFileException = (CorruptFileException) throwable; + listener.onFailure(new CorruptIndexException(corruptFileException.getMessage(), corruptFileException.getFileName())); + return; + } + listener.onFailure(ex); + }); + + completionListener = ActionListener.runBefore(completionListener, () -> { + try { + remoteTransferContainer.close(); + } catch (Exception e) { + logger.warn("Error occurred while closing streams", e); + } + }); + + WriteContext writeContext = remoteTransferContainer.createWriteContext(); + ((VerifyingMultiStreamBlobContainer) remoteDataDirectory.getBlobContainer()).asyncBlobUpload(writeContext, completionListener); + } + /** * This acquires a lock on a given commit by creating a lock file in lock directory using {@code FileLockInfo} * @param primaryTerm Primary Term of index at the time of commit. @@ -425,6 +527,10 @@ public void copyFrom(Directory from, String src, String dest, IOContext context, String remoteFilename; remoteFilename = getNewRemoteSegmentFilename(dest); remoteDataDirectory.copyFrom(from, src, remoteFilename, context); + postUpload(from, src, remoteFilename, checksum); + } + + private void postUpload(Directory from, String src, String remoteFilename, String checksum) throws IOException { UploadedSegmentMetadata segmentMetadata = new UploadedSegmentMetadata(src, remoteFilename, checksum, from.fileLength(src)); segmentsUploadedToRemoteStore.put(src, segmentMetadata); } @@ -528,6 +634,27 @@ private String getChecksumOfLocalFile(Directory directory, String file) throws I } } + private long calculateChecksumOfChecksum(Directory directory, String file) throws IOException { + try (IndexInput indexInput = directory.openInput(file, IOContext.DEFAULT)) { + long storedChecksum = CodecUtil.retrieveChecksum(indexInput); + CRC32 checksumOfChecksum = new CRC32(); + checksumOfChecksum.update(ByteUtils.toByteArrayBE(storedChecksum)); + try { + return JZlib.crc32_combine(storedChecksum, checksumOfChecksum.getValue(), SEGMENT_CHECKSUM_BYTES); + } catch (Exception e) { + throw new ChecksumCombinationException( + "Potentially corrupted file: Checksum combination failed while combining stored checksum " + + "and calculated checksum of stored checksum in segment file: " + + file + + ", directory: " + + directory, + file, + e + ); + } + } + } + private String getExistingRemoteFilename(String localFilename) { if (segmentsUploadedToRemoteStore.containsKey(localFilename)) { return segmentsUploadedToRemoteStore.get(localFilename).uploadedFilename; diff --git a/server/src/main/java/org/opensearch/index/store/exception/ChecksumCombinationException.java b/server/src/main/java/org/opensearch/index/store/exception/ChecksumCombinationException.java index a355473aa2afd..d8e1739fbaa9d 100644 --- a/server/src/main/java/org/opensearch/index/store/exception/ChecksumCombinationException.java +++ b/server/src/main/java/org/opensearch/index/store/exception/ChecksumCombinationException.java @@ -11,7 +11,7 @@ import org.apache.lucene.index.CorruptIndexException; /** - * Exception is raised when combination to two crc checksums fail. + * Exception is raised when combination of two CRC checksums fail. * * @opensearch.internal */ diff --git a/server/src/main/java/org/opensearch/index/translog/Checkpoint.java b/server/src/main/java/org/opensearch/index/translog/Checkpoint.java index 56de7e5daf55f..a9f905f52bc3a 100644 --- a/server/src/main/java/org/opensearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/opensearch/index/translog/Checkpoint.java @@ -233,7 +233,7 @@ public static void write(FileChannel fileChannel, Path checkpointFile, Checkpoin } } - private static byte[] createCheckpointBytes(Path checkpointFile, Checkpoint checkpoint) throws IOException { + public static byte[] createCheckpointBytes(Path checkpointFile, Checkpoint checkpoint) throws IOException { final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(V4_FILE_SIZE) { @Override public synchronized byte[] toByteArray() { diff --git a/server/src/main/java/org/opensearch/index/translog/checked/TranslogCheckedContainer.java b/server/src/main/java/org/opensearch/index/translog/TranslogCheckedContainer.java similarity index 74% rename from server/src/main/java/org/opensearch/index/translog/checked/TranslogCheckedContainer.java rename to server/src/main/java/org/opensearch/index/translog/TranslogCheckedContainer.java index b90794e29d2b1..7e2a38559166f 100644 --- a/server/src/main/java/org/opensearch/index/translog/checked/TranslogCheckedContainer.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogCheckedContainer.java @@ -6,13 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.translog.checked; +package org.opensearch.index.translog; -import org.opensearch.common.io.Channels; import org.opensearch.common.util.concurrent.ReleasableLock; -import java.io.IOException; -import java.nio.channels.FileChannel; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; import java.util.zip.CRC32; @@ -28,21 +25,15 @@ public class TranslogCheckedContainer { private final Checksum checksum; private final AtomicLong contentLength; private final ReleasableLock updateLock = new ReleasableLock(new ReentrantLock()); - private final String file; /** - * Creates TranslogCheckedContainer from provided channel. + * Create TranslogCheckedContainer from provided bytes * - * @param channel {@link FileChannel} to read from - * @param offset offset of channel from which bytes are to be read. - * @param len Length of bytes to be read. + * @param bytes The byte array to read from */ - public TranslogCheckedContainer(FileChannel channel, int offset, int len, String file) throws IOException { + public TranslogCheckedContainer(byte[] bytes) { this.checksum = new CRC32(); this.contentLength = new AtomicLong(); - this.file = file; - - byte[] bytes = Channels.readFromFileChannel(channel, offset, len); updateFromBytes(bytes, 0, bytes.length); } diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index af6ebcf7b7c66..8067cccb772a2 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -46,6 +46,7 @@ import java.io.EOFException; import java.io.IOException; +import java.io.OutputStream; import java.nio.channels.FileChannel; import java.nio.file.Path; @@ -213,12 +214,10 @@ private static void tryReportOldVersionError(final Path path, final FileChannel /** * Writes this header with the latest format into the file channel */ - void write(final FileChannel channel, boolean fsync) throws IOException { + void write(final OutputStream outputStream) throws IOException { // This output is intentionally not closed because closing it will close the FileChannel. @SuppressWarnings({ "IOResourceOpenedButNotSafelyClosed", "resource" }) - final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput( - new OutputStreamStreamOutput(java.nio.channels.Channels.newOutputStream(channel)) - ); + final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(new OutputStreamStreamOutput(outputStream)); CodecUtil.writeHeader(new OutputStreamDataOutput(out), TRANSLOG_CODEC, CURRENT_VERSION); // Write uuid final BytesRef uuid = new BytesRef(translogUUID); @@ -229,6 +228,11 @@ void write(final FileChannel channel, boolean fsync) throws IOException { // Checksum header out.writeInt((int) out.getChecksum()); out.flush(); + } + + void write(final FileChannel channel, boolean fsync) throws IOException { + OutputStream outputStream = java.nio.channels.Channels.newOutputStream(channel); + write(outputStream); if (fsync == true) { channel.force(true); } diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogReader.java b/server/src/main/java/org/opensearch/index/translog/TranslogReader.java index c4a4fb7a460a0..9ea3328587645 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogReader.java @@ -33,6 +33,7 @@ package org.opensearch.index.translog; import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.common.Nullable; import org.opensearch.common.io.Channels; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.SequenceNumbers; @@ -59,6 +60,11 @@ public class TranslogReader extends BaseTranslogReader implements Closeable { private final Checkpoint checkpoint; protected final AtomicBoolean closed = new AtomicBoolean(false); + @Nullable + private final Long translogChecksum; + @Nullable + private final Long checkpointChecksum; + /** * Create a translog writer against the specified translog file channel. * @@ -67,11 +73,34 @@ public class TranslogReader extends BaseTranslogReader implements Closeable { * @param path the path to the translog * @param header the header of the translog file */ - TranslogReader(final Checkpoint checkpoint, final FileChannel channel, final Path path, final TranslogHeader header) { + TranslogReader( + final Checkpoint checkpoint, + final FileChannel channel, + final Path path, + final TranslogHeader header, + final Long translogChecksum + ) throws IOException { super(checkpoint.generation, channel, path, header); this.length = checkpoint.offset; this.totalOperations = checkpoint.numOps; this.checkpoint = checkpoint; + this.translogChecksum = translogChecksum; + this.checkpointChecksum = (translogChecksum != null) ? calculateCheckpointChecksum(checkpoint, path) : null; + } + + private static Long calculateCheckpointChecksum(Checkpoint checkpoint, Path path) throws IOException { + TranslogCheckedContainer checkpointCheckedContainer = new TranslogCheckedContainer( + Checkpoint.createCheckpointBytes(path.getParent().resolve(Translog.CHECKPOINT_FILE_NAME), checkpoint) + ); + return checkpointCheckedContainer.getChecksum(); + } + + public Long getTranslogChecksum() { + return translogChecksum; + } + + public Long getCheckpointChecksum() { + return checkpointChecksum; } /** @@ -87,7 +116,7 @@ public class TranslogReader extends BaseTranslogReader implements Closeable { public static TranslogReader open(final FileChannel channel, final Path path, final Checkpoint checkpoint, final String translogUUID) throws IOException { final TranslogHeader header = TranslogHeader.read(translogUUID, path, channel); - return new TranslogReader(checkpoint, channel, path, header); + return new TranslogReader(checkpoint, channel, path, header, null); } /** @@ -115,9 +144,9 @@ TranslogReader closeIntoTrimmedReader(long aboveSeqNo, ChannelFactory channelFac IOUtils.fsync(checkpointFile.getParent(), true); - newReader = new TranslogReader(newCheckpoint, channel, path, header); + newReader = new TranslogReader(newCheckpoint, channel, path, header, translogChecksum); } else { - newReader = new TranslogReader(checkpoint, channel, path, header); + newReader = new TranslogReader(checkpoint, channel, path, header, translogChecksum); } toCloseOnFailure = null; return newReader; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index e19aece60adc0..e7b08b1dda3d2 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -37,6 +37,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; +import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.bytes.BytesReference; @@ -54,6 +55,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.ShardId; +import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; @@ -110,6 +112,9 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable { private final Map> seenSequenceNumbers; + @Nullable + private final TranslogCheckedContainer translogCheckedContainer; + private final Boolean remoteTranslogEnabled; private TranslogWriter( @@ -126,6 +131,7 @@ private TranslogWriter( final TragicExceptionHolder tragedy, final LongConsumer persistedSequenceNumberConsumer, final BigArrays bigArrays, + TranslogCheckedContainer translogCheckedContainer, Boolean remoteTranslogEnabled ) throws IOException { super(initialCheckpoint.generation, channel, path, header); @@ -151,6 +157,7 @@ private TranslogWriter( this.bigArrays = bigArrays; this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null; this.tragedy = tragedy; + this.translogCheckedContainer = translogCheckedContainer; this.remoteTranslogEnabled = remoteTranslogEnabled; } @@ -179,6 +186,12 @@ public static TranslogWriter create( checkpointChannel = channelFactory.open(checkpointFile, StandardOpenOption.WRITE); final TranslogHeader header = new TranslogHeader(translogUUID, primaryTerm); header.write(channel, !Boolean.TRUE.equals(remoteTranslogEnabled)); + TranslogCheckedContainer translogCheckedContainer = null; + if (Boolean.TRUE.equals(remoteTranslogEnabled)) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + header.write(byteArrayOutputStream); + translogCheckedContainer = new TranslogCheckedContainer(byteArrayOutputStream.toByteArray()); + } final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint( header.sizeInBytes(), fileGeneration, @@ -214,6 +227,7 @@ public static TranslogWriter create( tragedy, persistedSequenceNumberConsumer, bigArrays, + translogCheckedContainer, remoteTranslogEnabled ); } catch (Exception exception) { @@ -438,7 +452,13 @@ public TranslogReader closeIntoReader() throws IOException { closeWithTragicEvent(ex); throw ex; } - return new TranslogReader(getLastSyncedCheckpoint(), channel, path, header); + return new TranslogReader( + getLastSyncedCheckpoint(), + channel, + path, + header, + (translogCheckedContainer != null) ? translogCheckedContainer.getChecksum() : null + ); } else { throw new AlreadyClosedException( "translog [" + getGeneration() + "] is already closed (path [" + path + "]", @@ -571,6 +591,9 @@ private void writeAndReleaseOps(ReleasableBytesReference toWrite) throws IOExcep while (currentBytesConsumed != current.length) { int nBytesToWrite = Math.min(current.length - currentBytesConsumed, ioBuffer.remaining()); ioBuffer.put(current.bytes, current.offset + currentBytesConsumed, nBytesToWrite); + if (translogCheckedContainer != null) { + translogCheckedContainer.updateFromBytes(current.bytes, current.offset + currentBytesConsumed, nBytesToWrite); + } currentBytesConsumed += nBytesToWrite; if (ioBuffer.hasRemaining() == false) { ioBuffer.flip(); diff --git a/server/src/main/java/org/opensearch/index/translog/checked/package-info.java b/server/src/main/java/org/opensearch/index/translog/checked/package-info.java deleted file mode 100644 index ddb235fdbedce..0000000000000 --- a/server/src/main/java/org/opensearch/index/translog/checked/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** Contains checksum related utilities for translog files */ -package org.opensearch.index.translog.checked; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index d9feb1a832681..974e8af42b939 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -16,12 +16,22 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeFileInputStream; +import org.opensearch.index.translog.ChannelFactory; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.opensearch.common.blobstore.BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC; @@ -44,18 +54,18 @@ public BlobStoreTransferService(BlobStore blobStore, ThreadPool threadPool) { } @Override - public void uploadBlobAsync( - String threadpoolName, + public void uploadBlob( + String threadPoolName, final TransferFileSnapshot fileSnapshot, Iterable remoteTransferPath, - ActionListener listener + ActionListener listener, + WritePriority writePriority ) { assert remoteTransferPath instanceof BlobPath; BlobPath blobPath = (BlobPath) remoteTransferPath; - threadPool.executor(threadpoolName).execute(ActionRunnable.wrap(listener, l -> { - try (InputStream inputStream = fileSnapshot.inputStream()) { - blobStore.blobContainer(blobPath) - .writeBlobAtomic(fileSnapshot.getName(), inputStream, fileSnapshot.getContentLength(), true); + threadPool.executor(threadPoolName).execute(ActionRunnable.wrap(listener, l -> { + try { + uploadBlob(fileSnapshot, blobPath, writePriority); l.onResponse(fileSnapshot); } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), e); @@ -65,14 +75,84 @@ public void uploadBlobAsync( } @Override - public void uploadBlob(final TransferFileSnapshot fileSnapshot, Iterable remoteTransferPath) throws IOException { - assert remoteTransferPath instanceof BlobPath; + public void uploadBlob(final TransferFileSnapshot fileSnapshot, Iterable remoteTransferPath, WritePriority writePriority) + throws IOException { BlobPath blobPath = (BlobPath) remoteTransferPath; try (InputStream inputStream = fileSnapshot.inputStream()) { blobStore.blobContainer(blobPath).writeBlobAtomic(fileSnapshot.getName(), inputStream, fileSnapshot.getContentLength(), true); } } + @Override + public void uploadBlobs( + Set fileSnapshots, + final Map blobPaths, + ActionListener listener, + WritePriority writePriority + ) { + fileSnapshots.forEach(fileSnapshot -> { + BlobPath blobPath = blobPaths.get(fileSnapshot.getPrimaryTerm()); + if (!(blobStore.blobContainer(blobPath) instanceof VerifyingMultiStreamBlobContainer)) { + uploadBlob(ThreadPool.Names.TRANSLOG_TRANSFER, fileSnapshot, blobPath, listener, writePriority); + } else { + uploadBlob(fileSnapshot, listener, blobPath, writePriority); + } + }); + + } + + private void uploadBlob( + TransferFileSnapshot fileSnapshot, + ActionListener listener, + BlobPath blobPath, + WritePriority writePriority + ) { + + try { + ChannelFactory channelFactory = FileChannel::open; + long contentLength; + try (FileChannel channel = channelFactory.open(fileSnapshot.getPath(), StandardOpenOption.READ)) { + contentLength = channel.size(); + } + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + fileSnapshot.getName(), + fileSnapshot.getName(), + contentLength, + true, + writePriority, + (size, position) -> new OffsetRangeFileInputStream(fileSnapshot.getPath(), size, position), + Objects.requireNonNull(fileSnapshot.getChecksum()), + blobStore.blobContainer(blobPath) instanceof VerifyingMultiStreamBlobContainer + ); + ActionListener completionListener = ActionListener.wrap(resp -> listener.onResponse(fileSnapshot), ex -> { + logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), ex); + listener.onFailure(new FileTransferException(fileSnapshot, ex)); + }); + + completionListener = ActionListener.runBefore(completionListener, () -> { + try { + remoteTransferContainer.close(); + } catch (Exception e) { + logger.warn("Error occurred while closing streams", e); + } + }); + + WriteContext writeContext = remoteTransferContainer.createWriteContext(); + ((VerifyingMultiStreamBlobContainer) blobStore.blobContainer(blobPath)).asyncBlobUpload(writeContext, completionListener); + + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), e); + listener.onFailure(new FileTransferException(fileSnapshot, e)); + } finally { + try { + fileSnapshot.close(); + } catch (IOException e) { + logger.warn("Error while closing TransferFileSnapshot", e); + } + } + + } + @Override public InputStream downloadBlob(Iterable path, String fileName) throws IOException { return blobStore.blobContainer((BlobPath) path).readBlob(fileName); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java b/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java index 239ef7c3c9300..dcec94edd694f 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java @@ -107,10 +107,12 @@ public void close() throws IOException { public static class TransferFileSnapshot extends FileSnapshot { private final long primaryTerm; + private Long checksum; - public TransferFileSnapshot(Path path, long primaryTerm) throws IOException { + public TransferFileSnapshot(Path path, long primaryTerm, Long checksum) throws IOException { super(path); this.primaryTerm = primaryTerm; + this.checksum = checksum; } public TransferFileSnapshot(String name, byte[] content, long primaryTerm) throws IOException { @@ -118,6 +120,10 @@ public TransferFileSnapshot(String name, byte[] content, long primaryTerm) throw this.primaryTerm = primaryTerm; } + public Long getChecksum() { + return checksum; + } + public long getPrimaryTerm() { return primaryTerm; } @@ -148,8 +154,8 @@ public static final class TranslogFileSnapshot extends TransferFileSnapshot { private final long generation; - public TranslogFileSnapshot(long primaryTerm, long generation, Path path) throws IOException { - super(path, primaryTerm); + public TranslogFileSnapshot(long primaryTerm, long generation, Path path, Long checksum) throws IOException { + super(path, primaryTerm, checksum); this.generation = generation; } @@ -185,8 +191,9 @@ public static final class CheckpointFileSnapshot extends TransferFileSnapshot { private final long minTranslogGeneration; - public CheckpointFileSnapshot(long primaryTerm, long generation, long minTranslogGeneration, Path path) throws IOException { - super(path, primaryTerm); + public CheckpointFileSnapshot(long primaryTerm, long generation, long minTranslogGeneration, Path path, Long checksum) + throws IOException { + super(path, primaryTerm, checksum); this.minTranslogGeneration = minTranslogGeneration; this.generation = generation; } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java index 0e6496042e3d8..a240fd38cda11 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java @@ -10,11 +10,14 @@ import org.opensearch.action.ActionListener; import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -26,25 +29,40 @@ public interface TransferService { /** * Uploads the {@link TransferFileSnapshot} async, once the upload is complete the callback is invoked - * @param threadpoolName threadpool type which will be used to upload blobs asynchronously + * @param threadPoolName threadpool type which will be used to upload blobs asynchronously * @param fileSnapshot the file snapshot to upload * @param remotePath the remote path where upload should be made * @param listener the callback to be invoked once upload completes successfully/fails */ - void uploadBlobAsync( - String threadpoolName, + void uploadBlob( + String threadPoolName, final TransferFileSnapshot fileSnapshot, Iterable remotePath, - ActionListener listener + ActionListener listener, + WritePriority writePriority ); + /** + * Uploads multiple {@link TransferFileSnapshot}, once the upload is complete the callback is invoked + * @param fileSnapshots the file snapshots to upload + * @param blobPaths Primary term to {@link BlobPath} map + * @param listener the callback to be invoked once uploads complete successfully/fail + */ + void uploadBlobs( + Set fileSnapshots, + final Map blobPaths, + ActionListener listener, + WritePriority writePriority + ) throws Exception; + /** * Uploads the {@link TransferFileSnapshot} blob * @param fileSnapshot the file snapshot to upload * @param remotePath the remote path where upload should be made + * @param writePriority Priority by which content needs to be written. * @throws IOException the exception while transferring the data */ - void uploadBlob(final TransferFileSnapshot fileSnapshot, Iterable remotePath) throws IOException; + void uploadBlob(final TransferFileSnapshot fileSnapshot, Iterable remotePath, WritePriority writePriority) throws IOException; void deleteBlobs(Iterable path, List fileNames) throws IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java index b34c2282e874f..10dec13c81e1a 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java @@ -145,8 +145,14 @@ public TranslogCheckpointTransferSnapshot build() throws IOException { Path checkpointPath = location.resolve(checkpointGenFileNameMapper.apply(readerGeneration)); generations.add(readerGeneration); translogTransferSnapshot.add( - new TranslogFileSnapshot(readerPrimaryTerm, readerGeneration, translogPath), - new CheckpointFileSnapshot(readerPrimaryTerm, checkpointGeneration, minTranslogGeneration, checkpointPath) + new TranslogFileSnapshot(readerPrimaryTerm, readerGeneration, translogPath, reader.getTranslogChecksum()), + new CheckpointFileSnapshot( + readerPrimaryTerm, + checkpointGeneration, + minTranslogGeneration, + checkpointPath, + reader.getCheckpointChecksum() + ) ); if (readerGeneration > highestGeneration) { highestGeneration = readerGeneration; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 54140226e3744..0c63a7ffe4cce 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -17,6 +17,7 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -119,14 +120,16 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans }), latch ); + Map blobPathMap = new HashMap<>(); toUpload.forEach( - fileSnapshot -> transferService.uploadBlobAsync( - ThreadPool.Names.TRANSLOG_TRANSFER, - fileSnapshot, - remoteDataTransferPath.add(String.valueOf(fileSnapshot.getPrimaryTerm())), - latchedActionListener + fileSnapshot -> blobPathMap.put( + fileSnapshot.getPrimaryTerm(), + remoteDataTransferPath.add(String.valueOf(fileSnapshot.getPrimaryTerm())) ) ); + + transferService.uploadBlobs(toUpload, blobPathMap, latchedActionListener, WritePriority.HIGH); + try { if (latch.await(TRANSFER_TIMEOUT_IN_MILLIS, TimeUnit.MILLISECONDS) == false) { Exception ex = new TimeoutException("Timed out waiting for transfer of snapshot " + transferSnapshot + " to complete"); @@ -139,7 +142,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans throw ex; } if (exceptionList.isEmpty()) { - transferService.uploadBlob(prepareMetadata(transferSnapshot), remoteMetadataTransferPath); + transferService.uploadBlob(prepareMetadata(transferSnapshot), remoteMetadataTransferPath, WritePriority.HIGH); translogTransferListener.onUploadComplete(transferSnapshot); return true; } else { diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index ebc68c288e25a..d9f73a9b41658 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -46,10 +46,10 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.concurrent.XRejectedExecutionHandler; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; diff --git a/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java index 1ebec46042247..48940a0d401fd 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java @@ -9,6 +9,7 @@ package org.opensearch.common.blobstore.transfer; import org.junit.Before; +import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.StreamContext; import org.opensearch.common.blobstore.stream.write.WritePriority; @@ -21,6 +22,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.UUID; public class RemoteTransferContainerTests extends OpenSearchTestCase { @@ -140,6 +142,45 @@ public void testTypeOfProvidedStreamsAllCases() throws IOException { testTypeOfProvidedStreams(false); } + public void testCreateWriteContextAllCases() throws IOException { + testCreateWriteContext(true); + testCreateWriteContext(false); + } + + private void testCreateWriteContext(boolean doRemoteDataIntegrityCheck) throws IOException { + String remoteFileName = testFile.getFileName().toString() + UUID.randomUUID(); + Long expectedChecksum = randomLong(); + try ( + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + testFile.getFileName().toString(), + remoteFileName, + TEST_FILE_SIZE_BYTES, + true, + WritePriority.HIGH, + new RemoteTransferContainer.OffsetRangeInputStreamSupplier() { + @Override + public OffsetRangeInputStream get(long size, long position) throws IOException { + return new OffsetRangeFileInputStream(testFile, size, position); + } + }, + expectedChecksum, + doRemoteDataIntegrityCheck + ) + ) { + WriteContext writeContext = remoteTransferContainer.createWriteContext(); + assertEquals(remoteFileName, writeContext.getFileName()); + assertTrue(writeContext.isFailIfAlreadyExists()); + assertEquals(TEST_FILE_SIZE_BYTES, writeContext.getFileSize()); + assertEquals(WritePriority.HIGH, writeContext.getWritePriority()); + assertEquals(doRemoteDataIntegrityCheck, writeContext.doRemoteDataIntegrityCheck()); + if (doRemoteDataIntegrityCheck) { + assertEquals(expectedChecksum, writeContext.getExpectedChecksum()); + } else { + assertNull(writeContext.getExpectedChecksum()); + } + } + } + private void testTypeOfProvidedStreams(boolean isRemoteDataIntegritySupported) throws IOException { try ( RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index c37893877253e..ea092fffa3a9a 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -23,8 +23,12 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.junit.After; import org.junit.Before; +import org.mockito.Mockito; +import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -48,6 +52,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.HashMap; +import java.util.Collection; import java.util.concurrent.ExecutorService; import static org.mockito.Mockito.mock; @@ -68,6 +76,7 @@ public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase { private RemoteStoreMetadataLockManager mdLockManager; private RemoteSegmentStoreDirectory remoteSegmentStoreDirectory; + private TestUploadListener testUploadTracker; private IndexShard indexShard; private SegmentInfos segmentInfos; private ThreadPool threadPool; @@ -89,6 +98,7 @@ public void setup() throws IOException { mdLockManager, threadPool ); + testUploadTracker = new TestUploadListener(); Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT).build(); ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService(); @@ -503,6 +513,82 @@ public void testCopyFrom() throws IOException { storeDirectory.close(); } + public void testCopyFilesFromMultipart() throws Exception { + String filename = "_100.si"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = LuceneTestCase.newDirectory(); + IndexOutput indexOutput = storeDirectory.createOutput(filename, IOContext.DEFAULT); + indexOutput.writeString("Hello World!"); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + storeDirectory.sync(List.of(filename)); + + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); + Mockito.doAnswer(invocation -> { + ActionListener completionListener = invocation.getArgument(1); + completionListener.onResponse(null); + return null; + }).when(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + + CountDownLatch latch = new CountDownLatch(1); + ActionListener completionListener = new ActionListener() { + @Override + public void onResponse(Void unused) { + latch.countDown(); + } + + @Override + public void onFailure(Exception e) {} + }; + remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener); + assertTrue(latch.await(5000, TimeUnit.SECONDS)); + assertTrue(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + storeDirectory.close(); + } + + public void testCopyFilesFromMultipartIOException() throws Exception { + String filename = "_100.si"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = LuceneTestCase.newDirectory(); + IndexOutput indexOutput = storeDirectory.createOutput(filename, IOContext.DEFAULT); + indexOutput.writeString("Hello World!"); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + storeDirectory.sync(List.of(filename)); + + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); + Mockito.doAnswer(invocation -> { + ActionListener completionListener = invocation.getArgument(1); + completionListener.onFailure(new Exception("Test exception")); + return null; + }).when(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + CountDownLatch latch = new CountDownLatch(1); + ActionListener completionListener = new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) { + latch.countDown(); + } + }; + remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener); + assertTrue(latch.await(5000, TimeUnit.SECONDS)); + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + storeDirectory.close(); + } + public void testCopyFromException() throws IOException { String filename = "_100.si"; Directory storeDirectory = LuceneTestCase.newDirectory(); diff --git a/server/src/test/java/org/opensearch/index/store/TestUploadListener.java b/server/src/test/java/org/opensearch/index/store/TestUploadListener.java new file mode 100644 index 0000000000000..a2a61a93371e8 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/TestUploadListener.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.opensearch.common.util.UploadListener; + +import java.util.concurrent.ConcurrentHashMap; + +public class TestUploadListener implements UploadListener { + + private final ConcurrentHashMap uploadStatusMap = new ConcurrentHashMap<>(); + + enum UploadStatus { + BEFORE_UPLOAD, + UPLOAD_SUCCESS, + UPLOAD_FAILURE + } + + @Override + public void beforeUpload(String file) { + uploadStatusMap.put(file, UploadStatus.BEFORE_UPLOAD); + } + + @Override + public void onSuccess(String file) { + uploadStatusMap.put(file, UploadStatus.UPLOAD_SUCCESS); + } + + @Override + public void onFailure(String file) { + uploadStatusMap.put(file, UploadStatus.UPLOAD_FAILURE); + } + + public UploadStatus getUploadStatus(String file) { + return uploadStatusMap.get(file); + } +} diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java new file mode 100644 index 0000000000000..1175716679d0f --- /dev/null +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceMockRepositoryTests.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog.transfer; + +import org.mockito.Mockito; +import org.opensearch.action.ActionListener; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BlobStoreTransferServiceMockRepositoryTests extends OpenSearchTestCase { + + private ThreadPool threadPool; + + private BlobStore blobStore; + + @Override + public void setUp() throws Exception { + super.setUp(); + blobStore = mock(BlobStore.class); + threadPool = new TestThreadPool(getClass().getName()); + } + + public void testUploadBlobs() throws Exception { + Path testFile = createTempFile(); + Files.write(testFile, randomByteArrayOfLength(128), StandardOpenOption.APPEND); + FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( + testFile, + randomNonNegativeLong(), + 0L + ); + + VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + Mockito.doAnswer(invocation -> { + ActionListener completionListener = invocation.getArgument(1); + completionListener.onResponse(null); + return null; + }).when(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(blobContainer); + + TransferService transferService = new BlobStoreTransferService(blobStore, threadPool); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + AtomicReference exceptionRef = new AtomicReference<>(); + AtomicReference fileSnapshotRef = new AtomicReference<>(); + transferService.uploadBlobs(Collections.singleton(transferFileSnapshot), new HashMap<>() { + { + put(transferFileSnapshot.getPrimaryTerm(), new BlobPath().add("sample_path")); + } + }, new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(FileSnapshot.TransferFileSnapshot fileSnapshot) { + onResponseCalled.set(true); + fileSnapshotRef.set(fileSnapshot); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + } + }, latch), WritePriority.HIGH); + + assertTrue(latch.await(1000, TimeUnit.MILLISECONDS)); + verify(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + assertTrue(onResponseCalled.get()); + assertEquals(transferFileSnapshot.getPrimaryTerm(), fileSnapshotRef.get().getPrimaryTerm()); + assertEquals(transferFileSnapshot.getName(), fileSnapshotRef.get().getName()); + assertNull(exceptionRef.get()); + } + + public void testUploadBlobsIOException() throws Exception { + Path testFile = createTempFile(); + Files.write(testFile, randomByteArrayOfLength(128), StandardOpenOption.APPEND); + FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( + testFile, + randomNonNegativeLong(), + 0L + ); + + VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + doThrow(new IOException()).when(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(blobContainer); + + TransferService transferService = new BlobStoreTransferService(blobStore, threadPool); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + AtomicReference exceptionRef = new AtomicReference<>(); + transferService.uploadBlobs(Collections.singleton(transferFileSnapshot), new HashMap<>() { + { + put(transferFileSnapshot.getPrimaryTerm(), new BlobPath().add("sample_path")); + } + }, new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(FileSnapshot.TransferFileSnapshot fileSnapshot) { + onResponseCalled.set(true); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + } + }, latch), WritePriority.HIGH); + + assertTrue(latch.await(1000, TimeUnit.MILLISECONDS)); + verify(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + assertFalse(onResponseCalled.get()); + assertTrue(exceptionRef.get() instanceof FileTransferException); + } + + public void testUploadBlobsUploadFutureCompletedExceptionally() throws Exception { + Path testFile = createTempFile(); + Files.write(testFile, randomByteArrayOfLength(128), StandardOpenOption.APPEND); + FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( + testFile, + randomNonNegativeLong(), + 0L + ); + + VerifyingMultiStreamBlobContainer blobContainer = mock(VerifyingMultiStreamBlobContainer.class); + Mockito.doAnswer(invocation -> { + ActionListener completionListener = invocation.getArgument(1); + completionListener.onFailure(new Exception("Test exception")); + return null; + }).when(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + + when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(blobContainer); + + TransferService transferService = new BlobStoreTransferService(blobStore, threadPool); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + AtomicReference exceptionRef = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(FileSnapshot.TransferFileSnapshot fileSnapshot) { + onResponseCalled.set(true); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + } + }, latch); + transferService.uploadBlobs(Collections.singleton(transferFileSnapshot), new HashMap<>() { + { + put(transferFileSnapshot.getPrimaryTerm(), new BlobPath().add("sample_path")); + } + }, listener, WritePriority.HIGH); + + assertTrue(latch.await(1000, TimeUnit.MILLISECONDS)); + verify(blobContainer).asyncBlobUpload(any(WriteContext.class), any()); + assertFalse(onResponseCalled.get()); + assertTrue(exceptionRef.get() instanceof FileTransferException); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } +} diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java index 196fbd58c2c20..5502dc3089c62 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java @@ -12,6 +12,7 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; @@ -49,9 +50,13 @@ public void setUp() throws Exception { public void testUploadBlob() throws IOException { Path testFile = createTempFile(); Files.write(testFile, randomByteArrayOfLength(128), StandardOpenOption.APPEND); - FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot(testFile, randomNonNegativeLong()); + FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( + testFile, + randomNonNegativeLong(), + null + ); TransferService transferService = new BlobStoreTransferService(repository.blobStore(), threadPool); - transferService.uploadBlob(transferFileSnapshot, repository.basePath()); + transferService.uploadBlob(transferFileSnapshot, repository.basePath(), WritePriority.HIGH); } public void testUploadBlobFromByteArray() throws IOException { @@ -61,17 +66,21 @@ public void testUploadBlobFromByteArray() throws IOException { 1 ); TransferService transferService = new BlobStoreTransferService(repository.blobStore(), threadPool); - transferService.uploadBlob(transferFileSnapshot, repository.basePath()); + transferService.uploadBlob(transferFileSnapshot, repository.basePath(), WritePriority.NORMAL); } public void testUploadBlobAsync() throws IOException, InterruptedException { Path testFile = createTempFile(); Files.write(testFile, randomByteArrayOfLength(128), StandardOpenOption.APPEND); AtomicBoolean succeeded = new AtomicBoolean(false); - FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot(testFile, randomNonNegativeLong()); + FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( + testFile, + randomNonNegativeLong(), + null + ); CountDownLatch latch = new CountDownLatch(1); TransferService transferService = new BlobStoreTransferService(repository.blobStore(), threadPool); - transferService.uploadBlobAsync( + transferService.uploadBlob( ThreadPool.Names.TRANSLOG_TRANSFER, transferFileSnapshot, repository.basePath(), @@ -87,7 +96,8 @@ public void onResponse(FileSnapshot.TransferFileSnapshot fileSnapshot) { public void onFailure(Exception e) { throw new AssertionError("Failed to perform uploadBlobAsync", e); } - }, latch) + }, latch), + WritePriority.HIGH ); assertTrue(latch.await(1000, TimeUnit.MILLISECONDS)); assertTrue(succeeded.get()); diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/FileSnapshotTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/FileSnapshotTests.java index 6d2fb3794b107..8d07af5927135 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/FileSnapshotTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/FileSnapshotTests.java @@ -28,15 +28,15 @@ public void tearDown() throws Exception { public void testFileSnapshotPath() throws IOException { Path file = createTempFile(); Files.writeString(file, "hello"); - fileSnapshot = new FileSnapshot.TransferFileSnapshot(file, 12); + fileSnapshot = new FileSnapshot.TransferFileSnapshot(file, 12, null); assertFileSnapshotProperties(file); - try (FileSnapshot sameFileSnapshot = new FileSnapshot.TransferFileSnapshot(file, 12)) { + try (FileSnapshot sameFileSnapshot = new FileSnapshot.TransferFileSnapshot(file, 12, null)) { assertEquals(sameFileSnapshot, fileSnapshot); } - try (FileSnapshot sameFileDiffPTSnapshot = new FileSnapshot.TransferFileSnapshot(file, 34)) { + try (FileSnapshot sameFileDiffPTSnapshot = new FileSnapshot.TransferFileSnapshot(file, 34, null)) { assertNotEquals(sameFileDiffPTSnapshot, fileSnapshot); } } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java index be14e4a7bd380..fd0d44564ef6b 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java @@ -34,7 +34,8 @@ public void testOnSuccess() throws IOException { try ( FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( testFile, - randomNonNegativeLong() + randomNonNegativeLong(), + null ) ) { fileTransferTracker.onSuccess(transferFileSnapshot); @@ -58,11 +59,13 @@ public void testOnFailure() throws IOException { try ( FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( testFile, - randomNonNegativeLong() + randomNonNegativeLong(), + null ); FileSnapshot.TransferFileSnapshot transferFileSnapshot2 = new FileSnapshot.TransferFileSnapshot( testFile2, - randomNonNegativeLong() + randomNonNegativeLong(), + null ) ) { @@ -82,7 +85,8 @@ public void testUploaded() throws IOException { try ( FileSnapshot.TransferFileSnapshot transferFileSnapshot = new FileSnapshot.TransferFileSnapshot( testFile, - randomNonNegativeLong() + randomNonNegativeLong(), + null ); ) { diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 5f8aa64457896..66cd257299e25 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; import org.opensearch.index.translog.Translog; @@ -41,6 +42,8 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -80,20 +83,24 @@ public void tearDown() throws Exception { } @SuppressWarnings("unchecked") - public void testTransferSnapshot() throws IOException { + public void testTransferSnapshot() throws Exception { AtomicInteger fileTransferSucceeded = new AtomicInteger(); AtomicInteger fileTransferFailed = new AtomicInteger(); AtomicInteger translogTransferSucceeded = new AtomicInteger(); AtomicInteger translogTransferFailed = new AtomicInteger(); doNothing().when(transferService) - .uploadBlob(any(TransferFileSnapshot.class), Mockito.eq(remoteBaseTransferPath.add(String.valueOf(primaryTerm)))); + .uploadBlob( + any(TransferFileSnapshot.class), + Mockito.eq(remoteBaseTransferPath.add(String.valueOf(primaryTerm))), + any(WritePriority.class) + ); doAnswer(invocationOnMock -> { - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; - listener.onResponse((TransferFileSnapshot) invocationOnMock.getArguments()[1]); + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; + Set transferFileSnapshots = (Set) invocationOnMock.getArguments()[0]; + transferFileSnapshots.forEach(listener::onResponse); return null; - }).when(transferService) - .uploadBlobAsync(any(String.class), any(TransferFileSnapshot.class), any(BlobPath.class), any(ActionListener.class)); + }).when(transferService).uploadBlobs(anySet(), anyMap(), any(ActionListener.class), any(WritePriority.class)); FileTransferTracker fileTransferTracker = new FileTransferTracker(new ShardId("index", "indexUUid", 0)) { @Override @@ -145,13 +152,15 @@ public Set getCheckpointFileSnapshots() { primaryTerm, generation, minTranslogGeneration, - createTempFile(Translog.TRANSLOG_FILE_PREFIX + generation, Translog.CHECKPOINT_SUFFIX) + createTempFile(Translog.TRANSLOG_FILE_PREFIX + generation, Translog.CHECKPOINT_SUFFIX), + null ), new CheckpointFileSnapshot( primaryTerm, generation, minTranslogGeneration, - createTempFile(Translog.TRANSLOG_FILE_PREFIX + (generation - 1), Translog.CHECKPOINT_SUFFIX) + createTempFile(Translog.TRANSLOG_FILE_PREFIX + (generation - 1), Translog.CHECKPOINT_SUFFIX), + null ) ); } catch (IOException e) { @@ -166,12 +175,14 @@ public Set getTranslogFileSnapshots() { new TranslogFileSnapshot( primaryTerm, generation, - createTempFile(Translog.TRANSLOG_FILE_PREFIX + generation, Translog.TRANSLOG_FILE_SUFFIX) + createTempFile(Translog.TRANSLOG_FILE_PREFIX + generation, Translog.TRANSLOG_FILE_SUFFIX), + null ), new TranslogFileSnapshot( primaryTerm, generation - 1, - createTempFile(Translog.TRANSLOG_FILE_PREFIX + (generation - 1), Translog.TRANSLOG_FILE_SUFFIX) + createTempFile(Translog.TRANSLOG_FILE_PREFIX + (generation - 1), Translog.TRANSLOG_FILE_SUFFIX), + null ) ); } catch (IOException e) { From a07aac80d6e44e4963b5dffd3b86178135b11a4d Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Tue, 11 Jul 2023 08:31:56 -0700 Subject: [PATCH 101/186] [Mute] flaky RemoteStoreRefreshListenerTests (#8626) * Mute Flaky RemoteStoreRefreshListenerTests. Signed-off-by: Rishikesh1159 * Apply Spotless Check. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 --- .../opensearch/index/shard/RemoteStoreRefreshListenerTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 21a9393408529..ed1438cbc3b08 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.tests.store.BaseDirectoryWrapper; +import org.apache.lucene.tests.util.LuceneTestCase; import org.junit.After; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; @@ -47,6 +48,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8549") public class RemoteStoreRefreshListenerTests extends IndexShardTestCase { private IndexShard indexShard; private ClusterService clusterService; From 542041f98421cfc7fc8022167d8037db66bdb7c6 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Tue, 11 Jul 2023 22:05:08 +0530 Subject: [PATCH 102/186] Add logs for remote store metadata intermittent read failures (#8618) --------- Signed-off-by: bansvaru --- .../io/VersionedCodecStreamWrapper.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java b/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java index ff0af3954a3a3..9907e2225c64e 100644 --- a/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java +++ b/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java @@ -10,7 +10,11 @@ import java.io.IOException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -22,6 +26,8 @@ * @opensearch.internal */ public class VersionedCodecStreamWrapper { + private static final Logger logger = LogManager.getLogger(VersionedCodecStreamWrapper.class); + // TODO This can be updated to hold a streamReadWriteHandlerFactory and get relevant handler based on the stream versions private final IndexIOStreamHandler indexIOStreamHandler; private final int currentVersion; @@ -46,10 +52,21 @@ public VersionedCodecStreamWrapper(IndexIOStreamHandler indexIOStreamHandler, * @return stream content parsed into {@link T} */ public T readStream(IndexInput indexInput) throws IOException { - CodecUtil.checksumEntireFile(indexInput); - int readStreamVersion = checkHeader(indexInput); - T content = getHandlerForVersion(readStreamVersion).readContent(indexInput); - return content; + logger.debug("Reading input stream [{}] of length - [{}]", indexInput.toString(), indexInput.length()); + try { + CodecUtil.checksumEntireFile(indexInput); + int readStreamVersion = checkHeader(indexInput); + return getHandlerForVersion(readStreamVersion).readContent(indexInput); + } catch (CorruptIndexException cie) { + logger.error( + () -> new ParameterizedMessage( + "Error while validating header/footer for [{}]. Total data length [{}]", + indexInput.toString(), + indexInput.length() + ) + ); + throw cie; + } } /** From 4ccbf9d12f4cccae84a58f0264b718454e8a1e99 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 11 Jul 2023 09:39:01 -0700 Subject: [PATCH 103/186] Fix bug where ReplicationListeners would not complete on cancellation. (#8478) * [Segment Replication] Fix bug where ReplicationListeners would not complete on target cancellation. This change updates cancellation with Segment Replication to ensure all listeners are resolved. It does this by requesting cancellation before shard closure instead of using ReplicationCollection's cancelForShard which immediately removes it from the replicationCollection. This would cause the underlying ReplicationListener to never get invoked on close. This change includes new tests using suite scope to catch for any open tasks. This caught other locations where this was possible: 1. On a replica during force sync if the shard was closed while resolving its listeners, it would never call back to the primary if an exception was caught in the onDone method. - Fixed by refactoring those paths to use a ChannelActionListener and always reply to primary. 2. On the primary during forceSync, the primary would not successfully cancel before shard close during a forceSync, Fixed by wrapping the synchronous recoveryTarget::forceSync call in cancellableThreads. Signed-off-by: Marc Handalian PR cleanup. Signed-off-by: Marc Handalian Update log message Signed-off-by: Marc Handalian * PR feedback. Signed-off-by: Marc Handalian * Update server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java Co-authored-by: Suraj Singh Signed-off-by: Marc Handalian * Add more tests. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian Co-authored-by: Suraj Singh --- .../replication/SegmentReplicationBaseIT.java | 27 ++- .../SegmentReplicationSuiteIT.java | 88 ++++++++ .../recovery/RecoverySourceHandler.java | 2 +- .../replication/SegmentReplicationState.java | 11 +- .../replication/SegmentReplicationTarget.java | 182 +++++++--------- .../SegmentReplicationTargetService.java | 201 +++++++++++------- .../common/ReplicationCollection.java | 36 ++++ .../replication/common/ReplicationTarget.java | 1 + .../SegmentReplicationIndexShardTests.java | 124 ++++++++++- .../SegmentReplicationTargetServiceTests.java | 171 ++++++++++----- .../SegmentReplicationTargetTests.java | 6 +- .../recovery/ReplicationCollectionTests.java | 29 ++- 12 files changed, 623 insertions(+), 255 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 52fe85b51cebd..8fef88cbe6820 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -24,7 +24,9 @@ import org.opensearch.index.IndexService; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; @@ -158,6 +160,7 @@ protected void verifyStoreContent() throws Exception { final String indexName = primaryRouting.getIndexName(); final List replicaRouting = shardRoutingTable.replicaShards(); final IndexShard primaryShard = getIndexShard(clusterState, primaryRouting, indexName); + final int primaryDocCount = getDocCountFromShard(primaryShard); final Map primarySegmentMetadata = primaryShard.getSegmentMetadataMap(); for (ShardRouting replica : replicaRouting) { IndexShard replicaShard = getIndexShard(clusterState, replica, indexName); @@ -165,6 +168,8 @@ protected void verifyStoreContent() throws Exception { primarySegmentMetadata, replicaShard.getSegmentMetadataMap() ); + final int replicaDocCount = getDocCountFromShard(replicaShard); + assertEquals("Doc counts should match", primaryDocCount, replicaDocCount); if (recoveryDiff.missing.isEmpty() == false || recoveryDiff.different.isEmpty() == false) { fail( "Expected no missing or different segments between primary and replica but diff was missing: " @@ -185,10 +190,30 @@ protected void verifyStoreContent() throws Exception { }, 1, TimeUnit.MINUTES); } + private int getDocCountFromShard(IndexShard shard) { + try (final Engine.Searcher searcher = shard.acquireSearcher("test")) { + return searcher.getDirectoryReader().numDocs(); + } + } + private IndexShard getIndexShard(ClusterState state, ShardRouting routing, String indexName) { - return getIndexShard(state.nodes().get(routing.currentNodeId()).getName(), indexName); + return getIndexShard(state.nodes().get(routing.currentNodeId()).getName(), routing.shardId(), indexName); + } + + /** + * Fetch IndexShard by shardId, multiple shards per node allowed. + */ + protected IndexShard getIndexShard(String node, ShardId shardId, String indexName) { + final Index index = resolveIndex(indexName); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexServiceSafe(index); + final Optional id = indexService.shardIds().stream().filter(sid -> sid == shardId.id()).findFirst(); + return indexService.getShard(id.get()); } + /** + * Fetch IndexShard, assumes only a single shard per node. + */ protected IndexShard getIndexShard(String node, String indexName) { final Index index = resolveIndex(indexName); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java new file mode 100644 index 0000000000000..9025c1cc79884 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import org.junit.Before; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.OpenSearchIntegTestCase; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, minNumDataNodes = 2) +public class SegmentReplicationSuiteIT extends SegmentReplicationBaseIT { + + @Before + public void setup() { + internalCluster().startClusterManagerOnlyNode(); + createIndex(INDEX_NAME); + } + + @Override + public Settings indexSettings() { + final Settings.Builder builder = Settings.builder() + .put(super.indexSettings()) + // reset shard & replica count to random values set by OpenSearchIntegTestCase. + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas()) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); + + // TODO: Randomly enable remote store on these tests. + return builder.build(); + } + + public void testBasicReplication() throws Exception { + final int docCount = scaledRandomIntBetween(10, 200); + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(); + ensureGreen(INDEX_NAME); + verifyStoreContent(); + } + + public void testDropRandomNodeDuringReplication() throws Exception { + internalCluster().ensureAtLeastNumDataNodes(2); + internalCluster().startClusterManagerOnlyNodes(1); + + final int docCount = scaledRandomIntBetween(10, 200); + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(); + + internalCluster().restartRandomDataNode(); + + ensureYellow(INDEX_NAME); + client().prepareIndex(INDEX_NAME).setId(Integer.toString(docCount)).setSource("field", "value" + docCount).execute().get(); + internalCluster().startDataOnlyNode(); + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).actionGet(); + } + + public void testDeleteIndexWhileReplicating() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + final int docCount = scaledRandomIntBetween(10, 200); + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(INDEX_NAME); + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).actionGet(); + } + + public void testFullRestartDuringReplication() throws Exception { + internalCluster().startNode(); + final int docCount = scaledRandomIntBetween(10, 200); + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(INDEX_NAME); + internalCluster().fullRestart(); + ensureGreen(INDEX_NAME); + } +} diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 5e278f06cfb8f..0b343fb0b0871 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -835,7 +835,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis } else { // Force round of segment replication to update its checkpoint to primary's if (shard.indexSettings().isSegRepEnabled()) { - recoveryTarget.forceSegmentFileSync(); + cancellableThreads.execute(recoveryTarget::forceSegmentFileSync); } } stopWatch.stop(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 7a996ec7aedaa..226ccbaf01afa 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -45,8 +45,7 @@ public enum Stage { GET_CHECKPOINT_INFO((byte) 3), FILE_DIFF((byte) 4), GET_FILES((byte) 5), - FINALIZE_REPLICATION((byte) 6), - CANCELLED((byte) 7); + FINALIZE_REPLICATION((byte) 6); private static final Stage[] STAGES = new Stage[Stage.values().length]; @@ -245,14 +244,6 @@ public void setStage(Stage stage) { overallTimer.stop(); timingData.put("OVERALL", overallTimer.time()); break; - case CANCELLED: - if (this.stage == Stage.DONE) { - throw new IllegalStateException("can't move replication to Cancelled state from Done."); - } - this.stage = Stage.CANCELLED; - overallTimer.stop(); - timingData.put("OVERALL", overallTimer.time()); - break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 9d724d6cc9dcf..f59a7c2368689 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -12,16 +12,15 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ByteBuffersDataInput; import org.apache.lucene.store.ByteBuffersIndexInput; import org.apache.lucene.store.ChecksumIndexInput; -import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchCorruptionException; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; -import org.opensearch.common.CheckedConsumer; import org.opensearch.common.UUIDs; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; @@ -39,6 +38,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.List; +import java.util.Locale; /** * Represents the target of a replication event. @@ -102,17 +103,11 @@ public SegmentReplicationTarget retryCopy() { @Override public String description() { - return "Segment replication from " + source.toString(); + return String.format(Locale.ROOT, "Id:[%d] Shard:[%s] Source:[%s]", getId(), shardId(), source.getDescription()); } @Override public void notifyListener(ReplicationFailedException e, boolean sendShardFailure) { - // Cancellations still are passed to our SegmentReplicationListener as failures, if we have failed because of cancellation - // update the stage. - final Throwable cancelledException = ExceptionsHelper.unwrap(e, CancellableThreads.ExecutionCancelledException.class); - if (cancelledException != null) { - state.setStage(SegmentReplicationState.Stage.CANCELLED); - } listener.onFailure(state(), e, sendShardFailure); } @@ -141,141 +136,115 @@ public void writeFileChunk( /** * Start the Replication event. + * * @param listener {@link ActionListener} listener. */ public void startReplication(ActionListener listener) { cancellableThreads.setOnCancel((reason, beforeCancelEx) -> { - // This method only executes when cancellation is triggered by this node and caught by a call to checkForCancel, - // SegmentReplicationSource does not share CancellableThreads. - final CancellableThreads.ExecutionCancelledException executionCancelledException = - new CancellableThreads.ExecutionCancelledException("replication was canceled reason [" + reason + "]"); - notifyListener(new ReplicationFailedException("Segment replication failed", executionCancelledException), false); - throw executionCancelledException; + throw new CancellableThreads.ExecutionCancelledException("replication was canceled reason [" + reason + "]"); }); + // TODO: Remove this useless state. state.setStage(SegmentReplicationState.Stage.REPLICATING); final StepListener checkpointInfoListener = new StepListener<>(); final StepListener getFilesListener = new StepListener<>(); - final StepListener finalizeListener = new StepListener<>(); - cancellableThreads.checkForCancel(); - logger.trace("[shardId {}] Replica starting replication [id {}]", shardId().getId(), getId()); + logger.trace(new ParameterizedMessage("Starting Replication Target: {}", description())); // Get list of files to copy from this checkpoint. state.setStage(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO); + cancellableThreads.checkForCancel(); source.getCheckpointMetadata(getId(), checkpoint, checkpointInfoListener); - checkpointInfoListener.whenComplete(checkpointInfo -> getFiles(checkpointInfo, getFilesListener), listener::onFailure); - getFilesListener.whenComplete( - response -> finalizeReplication(checkpointInfoListener.result(), finalizeListener), - listener::onFailure - ); - finalizeListener.whenComplete(r -> listener.onResponse(null), listener::onFailure); + checkpointInfoListener.whenComplete(checkpointInfo -> { + final List filesToFetch = getFiles(checkpointInfo); + state.setStage(SegmentReplicationState.Stage.GET_FILES); + cancellableThreads.checkForCancel(); + source.getSegmentFiles(getId(), checkpointInfo.getCheckpoint(), filesToFetch, indexShard, getFilesListener); + }, listener::onFailure); + + getFilesListener.whenComplete(response -> { + finalizeReplication(checkpointInfoListener.result()); + listener.onResponse(null); + }, listener::onFailure); } - private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener getFilesListener) - throws IOException { + private List getFiles(CheckpointInfoResponse checkpointInfo) throws IOException { cancellableThreads.checkForCancel(); state.setStage(SegmentReplicationState.Stage.FILE_DIFF); final Store.RecoveryDiff diff = Store.segmentReplicationDiff(checkpointInfo.getMetadataMap(), indexShard.getSegmentMetadataMap()); - logger.trace("Replication diff for checkpoint {} {}", checkpointInfo.getCheckpoint(), diff); + logger.trace(() -> new ParameterizedMessage("Replication diff for checkpoint {} {}", checkpointInfo.getCheckpoint(), diff)); /* * Segments are immutable. So if the replica has any segments with the same name that differ from the one in the incoming * snapshot from source that means the local copy of the segment has been corrupted/changed in some way and we throw an * IllegalStateException to fail the shard */ if (diff.different.isEmpty() == false) { - IllegalStateException illegalStateException = new IllegalStateException( + throw new OpenSearchCorruptionException( new ParameterizedMessage( "Shard {} has local copies of segments that differ from the primary {}", indexShard.shardId(), diff.different ).getFormattedMessage() ); - ReplicationFailedException rfe = new ReplicationFailedException( - indexShard.shardId(), - "different segment files", - illegalStateException - ); - fail(rfe, true); - throw rfe; } for (StoreFileMetadata file : diff.missing) { state.getIndex().addFileDetail(file.name(), file.length(), false); } - // always send a req even if not fetching files so the primary can clear the copyState for this shard. - state.setStage(SegmentReplicationState.Stage.GET_FILES); - cancellableThreads.checkForCancel(); - source.getSegmentFiles(getId(), checkpointInfo.getCheckpoint(), diff.missing, indexShard, getFilesListener); + return diff.missing; } - private void finalizeReplication(CheckpointInfoResponse checkpointInfoResponse, ActionListener listener) { + private void finalizeReplication(CheckpointInfoResponse checkpointInfoResponse) throws OpenSearchCorruptionException { // TODO: Refactor the logic so that finalize doesn't have to be invoked for remote store as source if (source instanceof RemoteStoreReplicationSource) { - ActionListener.completeWith(listener, () -> { - state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); - return null; - }); + state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); return; } - ActionListener.completeWith(listener, () -> { - cancellableThreads.checkForCancel(); - state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); - Store store = null; + cancellableThreads.checkForCancel(); + state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); + Store store = null; + try { + store = store(); + store.incRef(); + store.buildInfosFromBytes( + multiFileWriter.getTempFileNames(), + checkpointInfoResponse.getInfosBytes(), + checkpointInfoResponse.getCheckpoint().getSegmentsGen(), + indexShard::finalizeReplication + ); + } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) { + // this is a fatal exception at this stage. + // this means we transferred files from the remote that have not be checksummed and they are + // broken. We have to clean up this shard entirely, remove all files and bubble it up to the + // source shard since this index might be broken there as well? The Source can handle this and checks + // its content on disk if possible. try { - store = store(); - store.incRef(); - CheckedConsumer finalizeReplication = indexShard::finalizeReplication; - store.buildInfosFromBytes( - multiFileWriter.getTempFileNames(), - checkpointInfoResponse.getInfosBytes(), - checkpointInfoResponse.getCheckpoint().getSegmentsGen(), - finalizeReplication - ); - } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) { - // this is a fatal exception at this stage. - // this means we transferred files from the remote that have not be checksummed and they are - // broken. We have to clean up this shard entirely, remove all files and bubble it up to the - // source shard since this index might be broken there as well? The Source can handle this and checks - // its content on disk if possible. try { - try { - store.removeCorruptionMarker(); - } finally { - Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files - } - } catch (Exception e) { - logger.debug("Failed to clean lucene index", e); - ex.addSuppressed(e); - } - ReplicationFailedException rfe = new ReplicationFailedException( - indexShard.shardId(), - "failed to clean after replication", - ex - ); - fail(rfe, true); - throw rfe; - } catch (OpenSearchException ex) { - /* - Ignore closed replication target as it can happen due to index shard closed event in a separate thread. - In such scenario, ignore the exception - */ - assert cancellableThreads.isCancelled() : "Replication target closed but segment replication not cancelled"; - logger.info("Replication target closed", ex); - } catch (Exception ex) { - ReplicationFailedException rfe = new ReplicationFailedException( - indexShard.shardId(), - "failed to clean after replication", - ex - ); - fail(rfe, true); - throw rfe; - } finally { - if (store != null) { - store.decRef(); + store.removeCorruptionMarker(); + } finally { + Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files } + } catch (Exception e) { + logger.debug("Failed to clean lucene index", e); + ex.addSuppressed(e); } - return null; - }); + throw new OpenSearchCorruptionException(ex); + } catch (AlreadyClosedException ex) { + // In this case the shard is closed at some point while updating the reader. + // This can happen when the engine is closed in a separate thread. + logger.warn("Shard is already closed, closing replication"); + } catch (OpenSearchException ex) { + /* + Ignore closed replication target as it can happen due to index shard closed event in a separate thread. + In such scenario, ignore the exception + */ + assert cancellableThreads.isCancelled() : "Replication target closed but segment replication not cancelled"; + } catch (Exception ex) { + throw new OpenSearchCorruptionException(ex); + } finally { + if (store != null) { + store.decRef(); + } + } } /** @@ -288,10 +257,15 @@ private ChecksumIndexInput toIndexInput(byte[] input) { ); } + /** + * Trigger a cancellation, this method will not close the target a subsequent call to #fail is required from target service. + */ @Override - protected void onCancel(String reason) { - cancellableThreads.cancel(reason); - source.cancel(); - multiFileWriter.close(); + public void cancel(String reason) { + if (finished.get() == false) { + logger.trace(new ParameterizedMessage("Cancelling replication for target {}", description())); + cancellableThreads.cancel(reason); + source.cancel(); + } } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index a7e0c0ec887ab..ac93412eef725 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -11,14 +11,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchCorruptionException; import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ChannelActionListener; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.CancellableThreads; +import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; @@ -43,10 +44,8 @@ import org.opensearch.transport.TransportResponse; import org.opensearch.transport.TransportService; -import java.io.IOException; import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import static org.opensearch.indices.replication.SegmentReplicationSourceService.Actions.UPDATE_VISIBLE_CHECKPOINT; @@ -145,7 +144,7 @@ public SegmentReplicationTargetService( @Override public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { if (indexShard != null && indexShard.indexSettings().isSegRepEnabled()) { - onGoingReplications.cancelForShard(shardId, "shard closed"); + onGoingReplications.requestCancel(indexShard.shardId(), "Shard closing"); latestReceivedCheckpoint.remove(shardId); } } @@ -167,7 +166,7 @@ public void afterIndexShardStarted(IndexShard indexShard) { @Override public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting oldRouting, ShardRouting newRouting) { if (oldRouting != null && indexShard.indexSettings().isSegRepEnabled() && oldRouting.primary() == false && newRouting.primary()) { - onGoingReplications.cancelForShard(indexShard.shardId(), "shard has been promoted to primary"); + onGoingReplications.requestCancel(indexShard.shardId(), "Shard has been promoted to primary"); latestReceivedCheckpoint.remove(indexShard.shardId()); } } @@ -224,11 +223,13 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe if (ongoingReplicationTarget != null) { if (ongoingReplicationTarget.getCheckpoint().getPrimaryTerm() < receivedCheckpoint.getPrimaryTerm()) { logger.trace( - "Cancelling ongoing replication from old primary with primary term {}", - ongoingReplicationTarget.getCheckpoint().getPrimaryTerm() + () -> new ParameterizedMessage( + "Cancelling ongoing replication {} from old primary with primary term {}", + ongoingReplicationTarget.description(), + ongoingReplicationTarget.getCheckpoint().getPrimaryTerm() + ) ); - onGoingReplications.cancel(ongoingReplicationTarget.getId(), "Cancelling stuck target after new primary"); - completedReplications.put(replicaShard.shardId(), ongoingReplicationTarget); + ongoingReplicationTarget.cancel("Cancelling stuck target after new primary"); } else { logger.trace( () -> new ParameterizedMessage( @@ -268,21 +269,20 @@ public void onReplicationFailure( ReplicationFailedException e, boolean sendShardFailure ) { - logger.trace( + logger.error( () -> new ParameterizedMessage( "[shardId {}] [replication id {}] Replication failed, timing data: {}", replicaShard.shardId().getId(), state.getReplicationId(), state.getTimingData() - ) + ), + e ); if (sendShardFailure == true) { - logger.error("replication failure", e); - replicaShard.failShard("replication failure", e); + failShard(e, replicaShard); } } }); - } } else { logger.trace( @@ -305,7 +305,14 @@ protected void updateVisibleCheckpoint(long replicationId, IndexShard replicaSha final TransportRequestOptions options = TransportRequestOptions.builder() .withTimeout(recoverySettings.internalActionTimeout()) .build(); - logger.debug("Updating replication checkpoint to {}", request.getCheckpoint()); + logger.trace( + () -> new ParameterizedMessage( + "Updating Primary shard that replica {}-{} is synced to checkpoint {}", + replicaShard.shardId(), + replicaShard.routingEntry().allocationId(), + request.getCheckpoint() + ) + ); RetryableTransportClient transportClient = new RetryableTransportClient( transportService, getPrimaryNode(primaryShard), @@ -315,19 +322,23 @@ protected void updateVisibleCheckpoint(long replicationId, IndexShard replicaSha final ActionListener listener = new ActionListener<>() { @Override public void onResponse(Void unused) { - logger.debug( - "Successfully updated replication checkpoint {} for replica {}", - replicaShard.shardId(), - request.getCheckpoint() + logger.trace( + () -> new ParameterizedMessage( + "Successfully updated replication checkpoint {} for replica {}", + replicaShard.shardId(), + request.getCheckpoint() + ) ); } @Override public void onFailure(Exception e) { logger.error( - "Failed to update visible checkpoint for replica {}, {}: {}", - replicaShard.shardId(), - request.getCheckpoint(), + () -> new ParameterizedMessage( + "Failed to update visible checkpoint for replica {}, {}:", + replicaShard.shardId(), + request.getCheckpoint() + ), e ); } @@ -350,6 +361,13 @@ private DiscoveryNode getPrimaryNode(ShardRouting primaryShard) { protected boolean processLatestReceivedCheckpoint(IndexShard replicaShard, Thread thread) { final ReplicationCheckpoint latestPublishedCheckpoint = latestReceivedCheckpoint.get(replicaShard.shardId()); if (latestPublishedCheckpoint != null && latestPublishedCheckpoint.isAheadOf(replicaShard.getLatestReplicationCheckpoint())) { + logger.trace( + () -> new ParameterizedMessage( + "Processing latest received checkpoint for shard {} {}", + replicaShard.shardId(), + latestPublishedCheckpoint + ) + ); Runnable runnable = () -> onNewCheckpoint(latestReceivedCheckpoint.get(replicaShard.shardId()), replicaShard); // Checks if we are using same thread and forks if necessary. if (thread == Thread.currentThread()) { @@ -381,7 +399,15 @@ public SegmentReplicationTarget startReplication(final IndexShard indexShard, fi // pkg-private for integration tests void startReplication(final SegmentReplicationTarget target) { - final long replicationId = onGoingReplications.start(target, recoverySettings.activityTimeout()); + final long replicationId; + try { + replicationId = onGoingReplications.startSafe(target, recoverySettings.activityTimeout()); + } catch (ReplicationFailedException e) { + // replication already running for shard. + target.fail(e, false); + return; + } + logger.trace(() -> new ParameterizedMessage("Added new replication to collection {}", target.description())); threadPool.generic().execute(new ReplicationRunner(replicationId)); } @@ -410,7 +436,7 @@ default void onFailure(ReplicationState state, ReplicationFailedException e, boo /** * Runnable implementation to trigger a replication event. */ - private class ReplicationRunner implements Runnable { + private class ReplicationRunner extends AbstractRunnable { final long replicationId; @@ -419,47 +445,49 @@ public ReplicationRunner(long replicationId) { } @Override - public void run() { + public void onFailure(Exception e) { + try (final ReplicationRef ref = onGoingReplications.get(replicationId)) { + logger.error(() -> new ParameterizedMessage("Error during segment replication, {}", ref.get().description()), e); + } + onGoingReplications.fail(replicationId, new ReplicationFailedException("Unexpected Error during replication", e), false); + } + + @Override + public void doRun() { start(replicationId); } } private void start(final long replicationId) { + final SegmentReplicationTarget target; try (ReplicationRef replicationRef = onGoingReplications.get(replicationId)) { // This check is for handling edge cases where the reference is removed before the ReplicationRunner is started by the // threadpool. if (replicationRef == null) { return; } - SegmentReplicationTarget target = onGoingReplications.getTarget(replicationId); - replicationRef.get().startReplication(new ActionListener<>() { - @Override - public void onResponse(Void o) { - onGoingReplications.markAsDone(replicationId); - if (target.state().getIndex().recoveredFileCount() != 0 && target.state().getIndex().recoveredBytes() != 0) { - completedReplications.put(target.shardId(), target); - } - + target = replicationRef.get(); + } + target.startReplication(new ActionListener<>() { + @Override + public void onResponse(Void o) { + logger.trace(() -> new ParameterizedMessage("Finished replicating {} marking as done.", target.description())); + onGoingReplications.markAsDone(replicationId); + if (target.state().getIndex().recoveredFileCount() != 0 && target.state().getIndex().recoveredBytes() != 0) { + completedReplications.put(target.shardId(), target); } + } - @Override - public void onFailure(Exception e) { - Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof CancellableThreads.ExecutionCancelledException) { - if (onGoingReplications.getTarget(replicationId) != null) { - IndexShard indexShard = onGoingReplications.getTarget(replicationId).indexShard(); - // if the target still exists in our collection, the primary initiated the cancellation, fail the replication - // but do not fail the shard. Cancellations initiated by this node from Index events will be removed with - // onGoingReplications.cancel and not appear in the collection when this listener resolves. - onGoingReplications.fail(replicationId, new ReplicationFailedException(indexShard, cause), false); - completedReplications.put(target.shardId(), target); - } - } else { - onGoingReplications.fail(replicationId, new ReplicationFailedException("Segment Replication failed", e), false); - } + @Override + public void onFailure(Exception e) { + logger.error(() -> new ParameterizedMessage("Exception replicating {} marking as failed.", target.description()), e); + if (e instanceof OpenSearchCorruptionException) { + onGoingReplications.fail(replicationId, new ReplicationFailedException("Store corruption during replication", e), true); + return; } - }); - } + onGoingReplications.fail(replicationId, new ReplicationFailedException("Segment Replication failed", e), false); + } + }); } private class FileChunkTransportRequestHandler implements TransportRequestHandler { @@ -484,27 +512,31 @@ public void messageReceived(final FileChunkRequest request, TransportChannel cha private class ForceSyncTransportRequestHandler implements TransportRequestHandler { @Override public void messageReceived(final ForceSyncRequest request, TransportChannel channel, Task task) throws Exception { - assert indicesService != null; - final IndexShard indexShard = indicesService.getShardOrNull(request.getShardId()); - // Proceed with round of segment replication only when it is allowed - if (indexShard == null || indexShard.getReplicationEngine().isEmpty()) { - logger.info("Ignore force segment replication sync as it is not allowed"); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - return; - } + forceReplication(request, new ChannelActionListener<>(channel, Actions.FORCE_SYNC, request)); + } + } + + private void forceReplication(ForceSyncRequest request, ActionListener listener) { + final ShardId shardId = request.getShardId(); + assert indicesService != null; + final IndexShard indexShard = indicesService.getShardOrNull(shardId); + // Proceed with round of segment replication only when it is allowed + if (indexShard == null || indexShard.getReplicationEngine().isEmpty()) { + listener.onResponse(TransportResponse.Empty.INSTANCE); + } else { startReplication(indexShard, new SegmentReplicationTargetService.SegmentReplicationListener() { @Override public void onReplicationDone(SegmentReplicationState state) { - logger.trace( - () -> new ParameterizedMessage( - "[shardId {}] [replication id {}] Replication complete to {}, timing data: {}", - indexShard.shardId().getId(), - state.getReplicationId(), - indexShard.getLatestReplicationCheckpoint(), - state.getTimingData() - ) - ); try { + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] [replication id {}] Force replication Sync complete to {}, timing data: {}", + shardId, + state.getReplicationId(), + indexShard.getLatestReplicationCheckpoint(), + state.getTimingData() + ) + ); // Promote engine type for primary target if (indexShard.recoveryState().getPrimary() == true) { indexShard.resetToWriteableEngine(); @@ -512,33 +544,40 @@ public void onReplicationDone(SegmentReplicationState state) { // Update the replica's checkpoint on primary's replication tracker. updateVisibleCheckpoint(state.getReplicationId(), indexShard); } - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } catch (InterruptedException | TimeoutException | IOException e) { - throw new RuntimeException(e); + listener.onResponse(TransportResponse.Empty.INSTANCE); + } catch (Exception e) { + logger.error("Error while marking replication completed", e); + listener.onFailure(e); } } @Override public void onReplicationFailure(SegmentReplicationState state, ReplicationFailedException e, boolean sendShardFailure) { - logger.trace( + logger.error( () -> new ParameterizedMessage( "[shardId {}] [replication id {}] Replication failed, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), state.getTimingData() - ) + ), + e ); - if (sendShardFailure == true) { - indexShard.failShard("replication failure", e); - } - try { - channel.sendResponse(e); - } catch (IOException ex) { - throw new RuntimeException(ex); + if (sendShardFailure) { + failShard(e, indexShard); } + listener.onFailure(e); } }); } } + private void failShard(ReplicationFailedException e, IndexShard indexShard) { + try { + indexShard.failShard("unrecoverable replication failure", e); + } catch (Exception inner) { + logger.error("Error attempting to fail shard", inner); + e.addSuppressed(inner); + } + } + } diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java index e918ac0a79691..c65ef27969154 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java @@ -70,6 +70,26 @@ public ReplicationCollection(Logger logger, ThreadPool threadPool) { this.threadPool = threadPool; } + /** + * Starts a new target event for a given shard, fails the given target if this shard is already replicating. + * @param target ReplicationTarget to start + * @param activityTimeout timeout for entire replication event + * @return The replication id + */ + public long startSafe(T target, TimeValue activityTimeout) { + synchronized (onGoingTargetEvents) { + final boolean isPresent = onGoingTargetEvents.values() + .stream() + .map(ReplicationTarget::shardId) + .anyMatch(t -> t.equals(target.shardId())); + if (isPresent) { + throw new ReplicationFailedException("Shard " + target.shardId() + " is already replicating"); + } else { + return start(target, activityTimeout); + } + } + } + /** * Starts a new target event for the given shard, source node and state * @@ -234,6 +254,22 @@ public boolean cancelForShard(ShardId shardId, String reason) { return cancelled; } + /** + * Trigger cancel on the target but do not remove it from the collection. + * This is intended to be called to ensure replication events are removed from the collection + * only when the target has closed. + * + * @param shardId {@link ShardId} shard events to cancel + * @param reason {@link String} reason for cancellation + */ + public void requestCancel(ShardId shardId, String reason) { + for (T value : onGoingTargetEvents.values()) { + if (value.shardId().equals(shardId)) { + value.cancel(reason); + } + } + } + /** * Get target for shard * diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java index 4d75ff4896706..344a4040be119 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java @@ -173,6 +173,7 @@ public void cancel(String reason) { public void fail(ReplicationFailedException e, boolean sendShardFailure) { if (finished.compareAndSet(false, true)) { try { + logger.debug("marking target " + description() + " as failed", e); notifyListener(e, sendShardFailure); } finally { try { diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index b3876a8ea8fd0..1f5980ba9bfe0 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.AlreadyClosedException; import org.junit.Assert; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; @@ -86,11 +87,12 @@ import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.spy; public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelReplicationTestCase { @@ -1156,6 +1158,125 @@ public void getSegmentFiles( } } + public void testCloseShardDuringFinalize() throws Exception { + try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard primary = shards.getPrimary(); + final IndexShard replica = shards.getReplicas().get(0); + final IndexShard replicaSpy = spy(replica); + + primary.refresh("Test"); + + doThrow(AlreadyClosedException.class).when(replicaSpy).finalizeReplication(any()); + + replicateSegments(primary, List.of(replicaSpy)); + } + } + + public void testCloseShardWhileGettingCheckpoint() throws Exception { + try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard primary = shards.getPrimary(); + final IndexShard replica = shards.getReplicas().get(0); + + primary.refresh("Test"); + + final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); + final SegmentReplicationTargetService targetService = newTargetService(sourceFactory); + SegmentReplicationSource source = new TestReplicationSource() { + + ActionListener listener; + + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + // set the listener, we will only fail it once we cancel the source. + this.listener = listener; + // shard is closing while we are copying files. + targetService.beforeIndexShardClosed(replica.shardId, replica, Settings.EMPTY); + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + ActionListener listener + ) { + Assert.fail("Unreachable"); + } + + @Override + public void cancel() { + // simulate listener resolving, but only after we have issued a cancel from beforeIndexShardClosed . + final RuntimeException exception = new CancellableThreads.ExecutionCancelledException("retryable action was cancelled"); + listener.onFailure(exception); + } + }; + when(sourceFactory.get(any())).thenReturn(source); + startReplicationAndAssertCancellation(replica, targetService); + + shards.removeReplica(replica); + closeShards(replica); + } + } + + public void testBeforeIndexShardClosedWhileCopyingFiles() throws Exception { + try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard primary = shards.getPrimary(); + final IndexShard replica = shards.getReplicas().get(0); + + primary.refresh("Test"); + + final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); + final SegmentReplicationTargetService targetService = newTargetService(sourceFactory); + SegmentReplicationSource source = new TestReplicationSource() { + + ActionListener listener; + + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + resolveCheckpointInfoResponseListener(listener, primary); + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + ActionListener listener + ) { + // set the listener, we will only fail it once we cancel the source. + this.listener = listener; + // shard is closing while we are copying files. + targetService.beforeIndexShardClosed(replica.shardId, replica, Settings.EMPTY); + } + + @Override + public void cancel() { + // simulate listener resolving, but only after we have issued a cancel from beforeIndexShardClosed . + final RuntimeException exception = new CancellableThreads.ExecutionCancelledException("retryable action was cancelled"); + listener.onFailure(exception); + } + }; + when(sourceFactory.get(any())).thenReturn(source); + startReplicationAndAssertCancellation(replica, targetService); + + shards.removeReplica(replica); + closeShards(replica); + } + } + public void testPrimaryCancelsExecution() throws Exception { try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { shards.startAll(); @@ -1245,7 +1366,6 @@ public void onReplicationDone(SegmentReplicationState state) { @Override public void onReplicationFailure(SegmentReplicationState state, ReplicationFailedException e, boolean sendShardFailure) { assertFalse(sendShardFailure); - assertEquals(SegmentReplicationState.Stage.CANCELLED, state.getStage()); latch.countDown(); } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index c632f2843cba2..32203c28c8ed8 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -8,8 +8,8 @@ package org.opensearch.indices.replication; +import org.apache.lucene.store.AlreadyClosedException; import org.junit.Assert; -import org.mockito.Mockito; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionListener; @@ -26,6 +26,7 @@ import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; +import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.ForceSyncRequest; @@ -49,6 +50,7 @@ import java.util.concurrent.TimeUnit; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -62,7 +64,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import static org.opensearch.indices.replication.SegmentReplicationState.Stage.CANCELLED; public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { @@ -240,71 +241,81 @@ public void testAlreadyOnNewCheckpoint() { verify(spy, times(0)).startReplication(any(), any()); } - public void testShardAlreadyReplicating() throws InterruptedException { - // Create a spy of Target Service so that we can verify invocation of startReplication call with specific checkpoint on it. - SegmentReplicationTargetService serviceSpy = spy(sut); - final SegmentReplicationTarget target = new SegmentReplicationTarget( - replicaShard, - replicationSource, - mock(SegmentReplicationTargetService.SegmentReplicationListener.class) - ); - // Create a Mockito spy of target to stub response of few method calls. - final SegmentReplicationTarget targetSpy = Mockito.spy(target); - CountDownLatch latch = new CountDownLatch(1); - // Mocking response when startReplication is called on targetSpy we send a new checkpoint to serviceSpy and later reduce countdown - // of latch. - doAnswer(invocation -> { - final ActionListener listener = invocation.getArgument(0); - // a new checkpoint arrives before we've completed. - serviceSpy.onNewCheckpoint(aheadCheckpoint, replicaShard); - listener.onResponse(null); - latch.countDown(); - return null; - }).when(targetSpy).startReplication(any()); - doNothing().when(targetSpy).onDone(); - - // start replication of this shard the first time. - serviceSpy.startReplication(targetSpy); + public void testShardAlreadyReplicating() { + sut.startReplication(replicaShard, mock(SegmentReplicationTargetService.SegmentReplicationListener.class)); + sut.startReplication(replicaShard, new SegmentReplicationTargetService.SegmentReplicationListener() { + @Override + public void onReplicationDone(SegmentReplicationState state) { + Assert.fail("Should not succeed"); + } - // wait for the new checkpoint to arrive, before the listener completes. - latch.await(30, TimeUnit.SECONDS); - verify(targetSpy, times(0)).cancel(any()); - verify(serviceSpy, times(0)).startReplication(eq(replicaShard), any()); + @Override + public void onReplicationFailure(SegmentReplicationState state, ReplicationFailedException e, boolean sendShardFailure) { + assertEquals("Shard " + replicaShard.shardId() + " is already replicating", e.getMessage()); + assertFalse(sendShardFailure); + } + }); } - public void testOnNewCheckpointFromNewPrimaryCancelOngoingReplication() throws IOException, InterruptedException { + public void testOnNewCheckpointFromNewPrimaryCancelOngoingReplication() throws InterruptedException { // Create a spy of Target Service so that we can verify invocation of startReplication call with specific checkpoint on it. SegmentReplicationTargetService serviceSpy = spy(sut); + doNothing().when(serviceSpy).updateVisibleCheckpoint(anyLong(), any()); + // skip post replication actions so we can assert execution counts. This will continue to process bc replica's pterm is not advanced + // post replication. + doReturn(true).when(serviceSpy).processLatestReceivedCheckpoint(any(), any()); // Create a Mockito spy of target to stub response of few method calls. - final SegmentReplicationTarget targetSpy = spy( - new SegmentReplicationTarget( - replicaShard, - replicationSource, - mock(SegmentReplicationTargetService.SegmentReplicationListener.class) - ) - ); CountDownLatch latch = new CountDownLatch(1); - // Mocking response when startReplication is called on targetSpy we send a new checkpoint to serviceSpy and later reduce countdown - // of latch. - doAnswer(invocation -> { - // short circuit loop on new checkpoint request - doReturn(null).when(serviceSpy).startReplication(eq(replicaShard), any()); - // a new checkpoint arrives before we've completed. - serviceSpy.onNewCheckpoint(newPrimaryCheckpoint, replicaShard); - try { - invocation.callRealMethod(); - } catch (CancellableThreads.ExecutionCancelledException e) { + SegmentReplicationSource source = new TestReplicationSource() { + + ActionListener listener; + + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + // set the listener, we will only fail it once we cancel the source. + this.listener = listener; latch.countDown(); + // do not resolve this listener yet, wait for cancel to hit. } - return null; - }).when(targetSpy).startReplication(any()); + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + ActionListener listener + ) { + Assert.fail("Unreachable"); + } + + @Override + public void cancel() { + // simulate listener resolving, but only after we have issued a cancel from beforeIndexShardClosed . + final RuntimeException exception = new CancellableThreads.ExecutionCancelledException("retryable action was cancelled"); + listener.onFailure(exception); + } + }; + + final SegmentReplicationTarget targetSpy = spy( + new SegmentReplicationTarget(replicaShard, source, mock(SegmentReplicationTargetService.SegmentReplicationListener.class)) + ); // start replication. This adds the target to on-ongoing replication collection serviceSpy.startReplication(targetSpy); + + // wait until we get to getCheckpoint step. latch.await(); - // wait for the new checkpoint to arrive, before the listener completes. - assertEquals(CANCELLED, targetSpy.state().getStage()); + + // new checkpoint arrives with higher pterm. + serviceSpy.onNewCheckpoint(newPrimaryCheckpoint, replicaShard); + + // ensure the old target is cancelled. and new iteration kicks off. verify(targetSpy, times(1)).cancel("Cancelling stuck target after new primary"); verify(serviceSpy, times(1)).startReplication(eq(replicaShard), any()); } @@ -467,6 +478,7 @@ public void testForceSegmentSyncHandler() throws Exception { } public void testForceSegmentSyncHandlerWithFailure() throws Exception { + allowShardFailures(); IndexShard spyReplicaShard = spy(replicaShard); ForceSyncRequest forceSyncRequest = new ForceSyncRequest(1L, 1L, replicaShard.shardId()); when(indicesService.getShardOrNull(forceSyncRequest.getShardId())).thenReturn(spyReplicaShard); @@ -488,4 +500,57 @@ public void testForceSegmentSyncHandlerWithFailure() throws Exception { assertTrue(nestedException instanceof IOException); assertTrue(nestedException.getMessage().contains("dummy failure")); } + + public void testForceSync_ShardDoesNotExist() { + ForceSyncRequest forceSyncRequest = new ForceSyncRequest(1L, 1L, new ShardId("no", "", 0)); + when(indicesService.getShardOrNull(forceSyncRequest.getShardId())).thenReturn(null); + transportService.submitRequest( + localNode, + SegmentReplicationTargetService.Actions.FORCE_SYNC, + forceSyncRequest, + TransportRequestOptions.builder().withTimeout(TRANSPORT_TIMEOUT).build(), + EmptyTransportResponseHandler.INSTANCE_SAME + ).txGet(); + } + + public void testForceSegmentSyncHandlerWithFailure_AlreadyClosedException_swallowed() throws Exception { + IndexShard spyReplicaShard = spy(replicaShard); + ForceSyncRequest forceSyncRequest = new ForceSyncRequest(1L, 1L, replicaShard.shardId()); + when(indicesService.getShardOrNull(forceSyncRequest.getShardId())).thenReturn(spyReplicaShard); + + AlreadyClosedException exception = new AlreadyClosedException("shard closed"); + doThrow(exception).when(spyReplicaShard).finalizeReplication(any()); + + // prevent shard failure to avoid test setup assertion + doNothing().when(spyReplicaShard).failShard(eq("replication failure"), any()); + transportService.submitRequest( + localNode, + SegmentReplicationTargetService.Actions.FORCE_SYNC, + forceSyncRequest, + TransportRequestOptions.builder().withTimeout(TRANSPORT_TIMEOUT).build(), + EmptyTransportResponseHandler.INSTANCE_SAME + ).txGet(); + } + + public void testTargetCancelledBeforeStartInvoked() { + final SegmentReplicationTarget target = new SegmentReplicationTarget( + replicaShard, + mock(SegmentReplicationSource.class), + new SegmentReplicationTargetService.SegmentReplicationListener() { + @Override + public void onReplicationDone(SegmentReplicationState state) { + Assert.fail(); + } + + @Override + public void onReplicationFailure(SegmentReplicationState state, ReplicationFailedException e, boolean sendShardFailure) { + // failures leave state object in last entered stage. + assertEquals(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO, state.getStage()); + assertTrue(e.getCause() instanceof CancellableThreads.ExecutionCancelledException); + } + } + ); + target.cancel("test"); + sut.startReplication(target); + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index ac8904527f7fb..4fb1edb4e496e 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -27,6 +27,7 @@ import org.junit.Assert; import org.mockito.Mockito; import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchCorruptionException; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; @@ -373,8 +374,9 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { - assert (e instanceof ReplicationFailedException); - assert (e.getMessage().contains("different segment files")); + assertTrue(e instanceof OpenSearchCorruptionException); + assertTrue(e.getMessage().contains("has local copies of segments that differ from the primary")); + segrepTarget.fail(new ReplicationFailedException(e), false); } }); } diff --git a/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java b/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java index 75ac1075e8ee0..0bb1bc210d946 100644 --- a/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java +++ b/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java @@ -38,12 +38,15 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.Store; +import org.opensearch.indices.replication.SegmentReplicationSource; +import org.opensearch.indices.replication.SegmentReplicationTarget; import org.opensearch.indices.replication.common.ReplicationCollection; import org.opensearch.indices.replication.common.ReplicationFailedException; import org.opensearch.indices.replication.common.ReplicationListener; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.recovery.RecoveryTarget; +import org.opensearch.indices.replication.common.ReplicationTarget; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -51,6 +54,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; +import static org.mockito.Mockito.mock; public class ReplicationCollectionTests extends OpenSearchIndexLevelReplicationTestCase { static final ReplicationListener listener = new ReplicationListener() { @@ -108,7 +112,30 @@ public void onFailure(ReplicationState state, ReplicationFailedException e, bool } } - public void testMultiReplicationsForSingleShard() throws Exception { + public void testStartMultipleReplicationsForSingleShard() throws Exception { + try (ReplicationGroup shards = createGroup(0)) { + shards.startAll(); + final ReplicationCollection collection = new ReplicationCollection<>(logger, threadPool); + final IndexShard shard = shards.addReplica(); + shards.recoverReplica(shard); + final SegmentReplicationTarget target1 = new SegmentReplicationTarget( + shard, + mock(SegmentReplicationSource.class), + mock(ReplicationListener.class) + ); + final SegmentReplicationTarget target2 = new SegmentReplicationTarget( + shard, + mock(SegmentReplicationSource.class), + mock(ReplicationListener.class) + ); + collection.startSafe(target1, TimeValue.timeValueMinutes(30)); + assertThrows(ReplicationFailedException.class, () -> collection.startSafe(target2, TimeValue.timeValueMinutes(30))); + target1.decRef(); + target2.decRef(); + } + } + + public void testGetReplicationTargetMultiReplicationsForSingleShard() throws Exception { try (ReplicationGroup shards = createGroup(0)) { final ReplicationCollection collection = new ReplicationCollection<>(logger, threadPool); final IndexShard shard1 = shards.addReplica(); From 0b42c2c0e41c8af6c527eace384d4ad4245a5a1f Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 12 Jul 2023 04:45:29 +1200 Subject: [PATCH 104/186] Bump org.apache.logging.log4j:log4j-core from 2.17.1 to 2.20.0 (#8307) - Use log4j's PluginProcessor annotation processor for plugin discovery - Remove usage of deprecated explicit plugin discovery - Pass through Configuration in OpenSearchJsonLayout to work around https://issues.apache.org/jira/browse/LOG4J2-3562 Signed-off-by: Thomas Signed-off-by: Daniel (dB.) Doubrovkine Signed-off-by: Thomas Farr Co-authored-by: Daniel (dB.) Doubrovkine Co-authored-by: Andrew Ross --- .gitignore | 5 ++- CHANGELOG.md | 1 + .../thirdPartyAudit/sample_jars/build.gradle | 2 +- buildSrc/version.properties | 3 +- libs/core/licenses/log4j-api-2.17.1.jar.sha1 | 1 - libs/core/licenses/log4j-api-2.20.0.jar.sha1 | 1 + .../licenses/log4j-1.2-api-2.17.1.jar.sha1 | 1 - .../licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 + .../licenses/log4j-1.2-api-2.17.1.jar.sha1 | 1 - .../licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 + .../licenses/log4j-1.2-api-2.17.1.jar.sha1 | 1 - .../licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 + .../licenses/log4j-1.2-api-2.17.1.jar.sha1 | 1 - .../licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 + .../licenses/log4j-slf4j-impl-2.17.1.jar.sha1 | 1 - .../licenses/log4j-slf4j-impl-2.20.0.jar.sha1 | 1 + .../licenses/log4j-1.2-api-2.17.1.jar.sha1 | 1 - .../licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 + qa/wildfly/build.gradle | 3 +- .../src/main/resources/log4j2.properties | 20 ----------- server/build.gradle | 5 +++ server/licenses/log4j-api-2.17.1.jar.sha1 | 1 - server/licenses/log4j-api-2.20.0.jar.sha1 | 1 + server/licenses/log4j-core-2.17.1.jar.sha1 | 1 - server/licenses/log4j-core-2.20.0.jar.sha1 | 1 + server/licenses/log4j-jul-2.17.1.jar.sha1 | 1 - server/licenses/log4j-jul-2.20.0.jar.sha1 | 1 + .../common/logging/LogConfigurator.java | 10 ------ .../common/logging/OpenSearchJsonLayout.java | 35 ++++++++++++++++--- test/framework/build.gradle | 3 ++ .../opensearch/test/OpenSearchTestCase.java | 2 -- 31 files changed, 57 insertions(+), 52 deletions(-) delete mode 100644 libs/core/licenses/log4j-api-2.17.1.jar.sha1 create mode 100644 libs/core/licenses/log4j-api-2.20.0.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.17.1.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.17.1.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.17.1.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/log4j-1.2-api-2.17.1.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.17.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.17.1.jar.sha1 create mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1 delete mode 100644 qa/wildfly/src/main/resources/log4j2.properties delete mode 100644 server/licenses/log4j-api-2.17.1.jar.sha1 create mode 100644 server/licenses/log4j-api-2.20.0.jar.sha1 delete mode 100644 server/licenses/log4j-core-2.17.1.jar.sha1 create mode 100644 server/licenses/log4j-core-2.20.0.jar.sha1 delete mode 100644 server/licenses/log4j-jul-2.17.1.jar.sha1 create mode 100644 server/licenses/log4j-jul-2.20.0.jar.sha1 diff --git a/.gitignore b/.gitignore index 9ab7de894636a..7514d55cc3c9a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ out/ benchmarks/src/main/generated/* benchmarks/bin/* benchmarks/build-eclipse-default/* +server/bin/* +server/build-eclipse-default/* +test/framework/build-eclipse-default/* # eclipse files .project @@ -61,4 +64,4 @@ testfixtures_shared/ .ci/jobs/ # build files generated -doc-tools/missing-doclet/bin/ +doc-tools/missing-doclet/bin/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index fc367be22c1fd..5f0d733bd7c55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,6 +143,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) - Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) - Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) ### Changed - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index b31e3b168c587..cb8050d1718c4 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -16,7 +16,7 @@ repositories { mavenCentral() } dependencies { - implementation 'org.apache.logging.log4j:log4j-core:2.20.0' + implementation "org.apache.logging.log4j:log4j-core:2.20.0" } ["0.0.1", "0.0.2"].forEach { v -> diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 7a2ddc24aabcf..246f6e3444224 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -14,8 +14,7 @@ jackson_databind = 2.15.2 snakeyaml = 2.0 icu4j = 70.1 supercsv = 2.4.0 -# Update to 2.17.2+ is breaking OpenSearchJsonLayout (see https://issues.apache.org/jira/browse/LOG4J2-3562) -log4j = 2.17.1 +log4j = 2.20.0 slf4j = 1.7.36 asm = 9.5 jettison = 1.5.4 diff --git a/libs/core/licenses/log4j-api-2.17.1.jar.sha1 b/libs/core/licenses/log4j-api-2.17.1.jar.sha1 deleted file mode 100644 index 9d0e5dc631ed5..0000000000000 --- a/libs/core/licenses/log4j-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d771af8e336e372fb5399c99edabe0919aeaf5b2 \ No newline at end of file diff --git a/libs/core/licenses/log4j-api-2.20.0.jar.sha1 b/libs/core/licenses/log4j-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..37154d9861ac0 --- /dev/null +++ b/libs/core/licenses/log4j-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +1fe6082e660daf07c689a89c94dc0f49c26b44bb \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.17.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.17.1.jar.sha1 deleted file mode 100644 index 23aa5c60bd596..0000000000000 --- a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db3a7e7f07e878b92ac4a8f1100bee8325d5713a \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..9829576d38ce0 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +689151374756cb809cb029f2501015bdc7733179 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.17.1.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.17.1.jar.sha1 deleted file mode 100644 index 23aa5c60bd596..0000000000000 --- a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db3a7e7f07e878b92ac4a8f1100bee8325d5713a \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..9829576d38ce0 --- /dev/null +++ b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +689151374756cb809cb029f2501015bdc7733179 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.17.1.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.17.1.jar.sha1 deleted file mode 100644 index 23aa5c60bd596..0000000000000 --- a/plugins/discovery-gce/licenses/log4j-1.2-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db3a7e7f07e878b92ac4a8f1100bee8325d5713a \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..9829576d38ce0 --- /dev/null +++ b/plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +689151374756cb809cb029f2501015bdc7733179 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/log4j-1.2-api-2.17.1.jar.sha1 b/plugins/repository-gcs/licenses/log4j-1.2-api-2.17.1.jar.sha1 deleted file mode 100644 index 23aa5c60bd596..0000000000000 --- a/plugins/repository-gcs/licenses/log4j-1.2-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db3a7e7f07e878b92ac4a8f1100bee8325d5713a \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..9829576d38ce0 --- /dev/null +++ b/plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +689151374756cb809cb029f2501015bdc7733179 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.17.1.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.17.1.jar.sha1 deleted file mode 100644 index 894ed8d886c3f..0000000000000 --- a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84692d456bcce689355d33d68167875e486954dd \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..800a4aa87ba0e --- /dev/null +++ b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1 @@ -0,0 +1 @@ +7ab4f082fd162f60afcaf2b8744a3d959feab3e8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.17.1.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.17.1.jar.sha1 deleted file mode 100644 index 23aa5c60bd596..0000000000000 --- a/plugins/repository-s3/licenses/log4j-1.2-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db3a7e7f07e878b92ac4a8f1100bee8325d5713a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..9829576d38ce0 --- /dev/null +++ b/plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +689151374756cb809cb029f2501015bdc7733179 \ No newline at end of file diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle index 391d2c78b489b..5d37be47e782e 100644 --- a/qa/wildfly/build.gradle +++ b/qa/wildfly/build.gradle @@ -58,11 +58,10 @@ dependencies { api "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-base:${versions.jackson}" api "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider:${versions.jackson}" api "com.github.fge:json-patch:1.9" - api "org.apache.logging.log4j:log4j-api:${versions.log4j}" - api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api(project(path: ':client:rest-high-level')) { exclude module: 'jakarta.annotation-api' } + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" testImplementation(project(':test:framework')) { exclude module: 'jakarta.annotation-api' } diff --git a/qa/wildfly/src/main/resources/log4j2.properties b/qa/wildfly/src/main/resources/log4j2.properties deleted file mode 100644 index 4fd5d69daf65b..0000000000000 --- a/qa/wildfly/src/main/resources/log4j2.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# - -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console diff --git a/server/build.gradle b/server/build.gradle index 38bbf020d860b..e80aa0f77a164 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -146,6 +146,7 @@ dependencies { api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-jul:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}", optional + annotationProcessor "org.apache.logging.log4j:log4j-core:${versions.log4j}" // jna api "net.java.dev.jna:jna:${versions.jna}" @@ -175,6 +176,10 @@ tasks.withType(JavaCompile).configureEach { options.compilerArgs -= '-Xlint:unchecked' } +compileJava { + options.compilerArgs += ['-processor', 'org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor'] +} + tasks.named("internalClusterTest").configure { // TODO: these run faster with C2 only because they run for so, so long jvmArgs -= '-XX:TieredStopAtLevel=1' diff --git a/server/licenses/log4j-api-2.17.1.jar.sha1 b/server/licenses/log4j-api-2.17.1.jar.sha1 deleted file mode 100644 index 9d0e5dc631ed5..0000000000000 --- a/server/licenses/log4j-api-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d771af8e336e372fb5399c99edabe0919aeaf5b2 \ No newline at end of file diff --git a/server/licenses/log4j-api-2.20.0.jar.sha1 b/server/licenses/log4j-api-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..37154d9861ac0 --- /dev/null +++ b/server/licenses/log4j-api-2.20.0.jar.sha1 @@ -0,0 +1 @@ +1fe6082e660daf07c689a89c94dc0f49c26b44bb \ No newline at end of file diff --git a/server/licenses/log4j-core-2.17.1.jar.sha1 b/server/licenses/log4j-core-2.17.1.jar.sha1 deleted file mode 100644 index 7d4634f3d4e18..0000000000000 --- a/server/licenses/log4j-core-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -779f60f3844dadc3ef597976fcb1e5127b1f343d \ No newline at end of file diff --git a/server/licenses/log4j-core-2.20.0.jar.sha1 b/server/licenses/log4j-core-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..49c972626563b --- /dev/null +++ b/server/licenses/log4j-core-2.20.0.jar.sha1 @@ -0,0 +1 @@ +eb2a9a47b1396e00b5eee1264296729a70565cc0 \ No newline at end of file diff --git a/server/licenses/log4j-jul-2.17.1.jar.sha1 b/server/licenses/log4j-jul-2.17.1.jar.sha1 deleted file mode 100644 index 4afb381a696e9..0000000000000 --- a/server/licenses/log4j-jul-2.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -881333b463d47828eda7443b19811763367b1916 \ No newline at end of file diff --git a/server/licenses/log4j-jul-2.20.0.jar.sha1 b/server/licenses/log4j-jul-2.20.0.jar.sha1 new file mode 100644 index 0000000000000..a456651e4569e --- /dev/null +++ b/server/licenses/log4j-jul-2.20.0.jar.sha1 @@ -0,0 +1 @@ +8170e6118eac1ab332046c179718a0f107f688e1 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java b/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java index 4438bf53fd62c..b1a338e65746f 100644 --- a/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java @@ -41,7 +41,6 @@ import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; import org.apache.logging.log4j.core.config.composite.CompositeConfiguration; -import org.apache.logging.log4j.core.config.plugins.util.PluginManager; import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory; import org.apache.logging.log4j.status.StatusConsoleListener; @@ -140,13 +139,6 @@ public static void configure(final Environment environment) throws IOException, configure(environment.settings(), environment.configDir(), environment.logsDir()); } - /** - * Load logging plugins so we can have {@code node_name} in the pattern. - */ - public static void loadLog4jPlugins() { - PluginManager.addPackage(LogConfigurator.class.getPackage().getName()); - } - /** * Sets the node name. This is called before logging is configured if the * node name is set in opensearch.yml. Otherwise it is called as soon @@ -172,8 +164,6 @@ private static void configure(final Settings settings, final Path configsPath, f Objects.requireNonNull(configsPath); Objects.requireNonNull(logsPath); - loadLog4jPlugins(); - setLogConfigurationSystemProperty(logsPath, settings); // we initialize the status logger immediately otherwise Log4j will complain when we try to get the context configureStatusLogger(); diff --git a/server/src/main/java/org/opensearch/common/logging/OpenSearchJsonLayout.java b/server/src/main/java/org/opensearch/common/logging/OpenSearchJsonLayout.java index 5896aec0ce71c..60df94036cc6b 100644 --- a/server/src/main/java/org/opensearch/common/logging/OpenSearchJsonLayout.java +++ b/server/src/main/java/org/opensearch/common/logging/OpenSearchJsonLayout.java @@ -34,10 +34,12 @@ import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.Node; import org.apache.logging.log4j.core.config.plugins.Plugin; import org.apache.logging.log4j.core.config.plugins.PluginAttribute; import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; import org.apache.logging.log4j.core.config.plugins.PluginFactory; import org.apache.logging.log4j.core.layout.AbstractStringLayout; import org.apache.logging.log4j.core.layout.ByteBufferDestination; @@ -94,11 +96,18 @@ public class OpenSearchJsonLayout extends AbstractStringLayout { private final PatternLayout patternLayout; - protected OpenSearchJsonLayout(String typeName, Charset charset, String[] opensearchMessageFields, int maxMessageLength) { + protected OpenSearchJsonLayout( + String typeName, + Charset charset, + String[] opensearchMessageFields, + int maxMessageLength, + Configuration configuration + ) { super(charset); this.patternLayout = PatternLayout.newBuilder() .withPattern(pattern(typeName, opensearchMessageFields, maxMessageLength)) .withAlwaysWriteExceptions(false) + .withConfiguration(configuration) .build(); } @@ -173,8 +182,14 @@ private String inQuotes(String s) { } @PluginFactory - public static OpenSearchJsonLayout createLayout(String type, Charset charset, String[] opensearchmessagefields, int maxMessageLength) { - return new OpenSearchJsonLayout(type, charset, opensearchmessagefields, maxMessageLength); + public static OpenSearchJsonLayout createLayout( + String type, + Charset charset, + String[] opensearchmessagefields, + int maxMessageLength, + Configuration configuration + ) { + return new OpenSearchJsonLayout(type, charset, opensearchmessagefields, maxMessageLength, configuration); } PatternLayout getPatternLayout() { @@ -202,6 +217,9 @@ public static class Builder> extends A @PluginAttribute(value = "maxmessagelength", defaultInt = 10000) private int maxMessageLength; + @PluginConfiguration + private Configuration configuration; + public Builder() { setCharset(StandardCharsets.UTF_8); setMaxMessageLength(10000); @@ -210,7 +228,7 @@ public Builder() { @Override public OpenSearchJsonLayout build() { String[] split = Strings.isNullOrEmpty(opensearchMessageFields) ? new String[] {} : opensearchMessageFields.split(","); - return OpenSearchJsonLayout.createLayout(type, charset, split, maxMessageLength); + return OpenSearchJsonLayout.createLayout(type, charset, split, maxMessageLength, configuration); } public Charset getCharset() { @@ -248,6 +266,15 @@ public B setMaxMessageLength(final int maxMessageLength) { this.maxMessageLength = maxMessageLength; return asBuilder(); } + + public Configuration getConfiguration() { + return configuration; + } + + public B setConfiguration(final Configuration configuration) { + this.configuration = configuration; + return asBuilder(); + } } @PluginBuilderFactory diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 2532fdf1938fd..c65bf51c6af36 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -49,11 +49,14 @@ dependencies { api "org.mockito:mockito-core:${versions.mockito}" api "net.bytebuddy:byte-buddy:${versions.bytebuddy}" api "org.objenesis:objenesis:${versions.objenesis}" + + annotationProcessor "org.apache.logging.log4j:log4j-core:${versions.log4j}" } compileJava.options.compilerArgs -= '-Xlint:cast' compileJava.options.compilerArgs -= '-Xlint:rawtypes' compileJava.options.compilerArgs -= '-Xlint:unchecked' +compileJava.options.compilerArgs += ['-processor', 'org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor'] compileTestJava.options.compilerArgs -= '-Xlint:rawtypes' // the main files are actually test files, so use the appropriate forbidden api sigs diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index ec397a2baa640..1bea22852df4c 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -83,7 +83,6 @@ import org.opensearch.common.logging.DeprecatedMessage; import org.opensearch.common.logging.HeaderWarning; import org.opensearch.common.logging.HeaderWarningAppender; -import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.logging.Loggers; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -239,7 +238,6 @@ public void tearDown() throws Exception { static { TEST_WORKER_VM_ID = System.getProperty(TEST_WORKER_SYS_PROPERTY, DEFAULT_TEST_WORKER_ID); setTestSysProps(); - LogConfigurator.loadLog4jPlugins(); String leakLoggerName = "io.netty.util.ResourceLeakDetector"; Logger leakLogger = LogManager.getLogger(leakLoggerName); From 3d7d33bdbc02ec8e363c5394a9d89451f76e979a Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Tue, 11 Jul 2023 10:01:08 -0700 Subject: [PATCH 105/186] [Segment Replication] Add logic back to update tracking replication checkpoint on source (#8560) * [Segment Replication] Add logic back to update tracking replication checkpoint on source Signed-off-by: Suraj Singh * Update comment Signed-off-by: Suraj Singh * Address review comments & mute breaking bwc-test Signed-off-by: Suraj Singh * Spotless check Signed-off-by: Suraj Singh * Stop timer inside try to prevent double stop on timer Signed-off-by: Suraj Singh * Update PressureITs to wait for appropriate transport call for replica update Signed-off-by: Suraj Singh * Spotless check Signed-off-by: Suraj Singh --------- Signed-off-by: Suraj Singh --- .../java/org/opensearch/upgrades/IndexingIT.java | 1 + .../replication/SegmentReplicationBaseIT.java | 12 +++++++++++- .../indices/replication/SegmentReplicationIT.java | 6 ------ .../replication/OngoingSegmentReplications.java | 6 +----- .../replication/SegmentReplicationSourceHandler.java | 12 +++++++++++- .../replication/SegmentReplicationTargetService.java | 5 +++++ .../SegmentReplicationSourceHandlerTests.java | 7 +++++-- 7 files changed, 34 insertions(+), 15 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index 93c0bc96a5183..b60ee09d39048 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -247,6 +247,7 @@ public void testIndexing() throws IOException, ParseException { * * @throws Exception */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8322") public void testIndexingWithSegRep() throws Exception { if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) { logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 8fef88cbe6820..4692210ccc577 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -19,6 +19,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; @@ -222,6 +223,11 @@ protected IndexShard getIndexShard(String node, String indexName) { return indexService.getShard(shardId.get()); } + protected boolean segmentReplicationWithRemoteEnabled() { + return IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings()).booleanValue() + && "true".equalsIgnoreCase(featureFlagSettings().get(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL)); + } + protected Releasable blockReplication(List nodes, CountDownLatch latch) { CountDownLatch pauseReplicationLatch = new CountDownLatch(nodes.size()); for (String node : nodes) { @@ -231,7 +237,11 @@ protected Releasable blockReplication(List nodes, CountDownLatch latch) node )); mockTargetTransportService.addSendBehavior((connection, requestId, action, request, options) -> { - if (action.equals(SegmentReplicationSourceService.Actions.UPDATE_VISIBLE_CHECKPOINT)) { + String actionToWaitFor = SegmentReplicationSourceService.Actions.GET_SEGMENT_FILES; + if (segmentReplicationWithRemoteEnabled()) { + actionToWaitFor = SegmentReplicationSourceService.Actions.UPDATE_VISIBLE_CHECKPOINT; + } + if (action.equals(actionToWaitFor)) { try { latch.countDown(); pauseReplicationLatch.await(); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 873c05843fb56..099c15267c2f7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -49,7 +49,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.lease.Releasable; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexModule; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; @@ -1324,9 +1323,4 @@ public void testPrimaryReceivesDocsDuringReplicaRecovery() throws Exception { ensureGreen(INDEX_NAME); waitForSearchableDocs(2, nodes); } - - private boolean segmentReplicationWithRemoteEnabled() { - return IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings()).booleanValue() - && "true".equalsIgnoreCase(featureFlagSettings().get(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL)); - } } diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java index 050a66bedcf5d..f32887175d4f3 100644 --- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java +++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java @@ -120,11 +120,7 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener listener) { + // Short circuit when no files to transfer + if (request.getFilesToFetch().isEmpty()) { + // before completion, alert the primary of the replica's state. + shard.updateVisibleCheckpointForShard(request.getTargetAllocationId(), copyState.getCheckpoint()); + listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); + return; + } + final ReplicationTimer timer = new ReplicationTimer(); if (isReplicating.compareAndSet(false, true) == false) { throw new OpenSearchException("Replication to {} is already running.", shard.shardId()); @@ -159,10 +168,11 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene sendFileStep.whenComplete(r -> { try { + shard.updateVisibleCheckpointForShard(allocationId, copyState.getCheckpoint()); future.onResponse(new GetSegmentFilesResponse(List.of(storeFileMetadata))); + timer.stop(); } finally { IOUtils.close(resources); - timer.stop(); logger.trace( "[replication id {}] Source node completed sending files to target node [{}], timing: {}", request.getReplicationId(), diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index ac93412eef725..467f499056345 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -292,6 +292,11 @@ public void onReplicationFailure( } protected void updateVisibleCheckpoint(long replicationId, IndexShard replicaShard) { + // Update replication checkpoint on source via transport call only supported for remote store integration. For node- + // node communication, checkpoint update is piggy-backed to GET_SEGMENT_FILES transport call + if (replicaShard.indexSettings().isRemoteStoreEnabled() == false) { + return; + } ShardRouting primaryShard = clusterService.state().routingTable().shardRoutingTable(replicaShard.shardId()).primaryShard(); final UpdateVisibleCheckpointRequest request = new UpdateVisibleCheckpointRequest( diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java index 607f9dd91e35e..b4e9166f377ec 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java @@ -196,11 +196,13 @@ public void testReplicationAlreadyRunning() throws IOException { 1 ); + final List expectedFiles = List.of(new StoreFileMetadata("_0.si", 20, "test", Version.CURRENT.luceneVersion)); + final GetSegmentFilesRequest getSegmentFilesRequest = new GetSegmentFilesRequest( 1L, replica.routingEntry().allocationId().getId(), replicaDiscoveryNode, - Collections.emptyList(), + expectedFiles, latestReplicationCheckpoint ); @@ -224,11 +226,12 @@ public void testCancelReplication() throws IOException, InterruptedException { 1 ); + final List expectedFiles = List.of(new StoreFileMetadata("_0.si", 20, "test", Version.CURRENT.luceneVersion)); final GetSegmentFilesRequest getSegmentFilesRequest = new GetSegmentFilesRequest( 1L, replica.routingEntry().allocationId().getId(), replicaDiscoveryNode, - Collections.emptyList(), + expectedFiles, latestReplicationCheckpoint ); From 2004ba09eec681175a94f289bb472399db08aa86 Mon Sep 17 00:00:00 2001 From: Mingshi Liu <113382730+mingshl@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:00:20 -0700 Subject: [PATCH 106/186] [Search pipelines] Add Global Ignore_failure options for Processors (#8373) * Add Global Ingore_failure options for Processors Signed-off-by: Mingshi Liu * add changelog Signed-off-by: Mingshi Liu * Add ignore_failure to 40_rename_response Signed-off-by: Mingshi Liu * Change Boolean to boolean and refactor AbstractProcessor Signed-off-by: Mingshi Liu * rename to isIgnoreFailure and add tests Signed-off-by: Mingshi Liu * rename to isIgnoreFailure and add tests Signed-off-by: Mingshi Liu * add ignoreFailure to runSearchPhaseResultsTransformer Signed-off-by: Mingshi Liu * fix filter query and change log warn message Signed-off-by: Mingshi Liu * Add test on matching each processor stat Signed-off-by: Mingshi Liu * Add test on matching each processor stat Signed-off-by: Mingshi Liu * remove extra spaces and words Signed-off-by: Mingshi Liu * use IGNORE_FAILURE_KEY Signed-off-by: Mingshi Liu --------- Signed-off-by: Mingshi Liu --- CHANGELOG.md | 1 + .../common/FilterQueryRequestProcessor.java | 13 +- .../common/RenameFieldResponseProcessor.java | 24 +- .../common/ScriptRequestProcessor.java | 8 +- .../FilterQueryRequestProcessorTests.java | 6 +- .../RenameFieldResponseProcessorTests.java | 7 +- .../common/ScriptRequestProcessorTests.java | 11 +- .../search_pipeline/40_rename_response.yml | 33 +- .../opensearch/ingest/ConfigurationUtils.java | 8 +- .../search/pipeline}/AbstractProcessor.java | 15 +- .../opensearch/search/pipeline/Pipeline.java | 50 ++- .../search/pipeline/PipelineWithMetrics.java | 6 +- .../opensearch/search/pipeline/Processor.java | 6 + .../pipeline/SearchPipelineServiceTests.java | 379 +++++++++++++----- 14 files changed, 440 insertions(+), 127 deletions(-) rename {modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common => server/src/main/java/org/opensearch/search/pipeline}/AbstractProcessor.java (60%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f0d733bd7c55..0de0d823aa764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) - Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) - Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) +- [Search pipelines] Add Global Ignore_failure options for Processors ([#8373](https://github.com/opensearch-project/OpenSearch/pull/8373)) - Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) - Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java index d8862aa59cede..c776181a64b26 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java @@ -21,6 +21,7 @@ import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.AbstractProcessor; import org.opensearch.search.pipeline.SearchRequestProcessor; import java.io.InputStream; @@ -53,12 +54,13 @@ public String getType() { /** * Constructor that takes a filter query. * - * @param tag processor tag - * @param description processor description + * @param tag processor tag + * @param description processor description + * @param ignoreFailure option to ignore failure * @param filterQuery the query that will be added as a filter to incoming queries */ - public FilterQueryRequestProcessor(String tag, String description, QueryBuilder filterQuery) { - super(tag, description); + FilterQueryRequestProcessor(String tag, String description, boolean ignoreFailure, QueryBuilder filterQuery) { + super(tag, description, ignoreFailure); this.filterQuery = filterQuery; } @@ -101,6 +103,7 @@ public FilterQueryRequestProcessor create( Map> processorFactories, String tag, String description, + boolean ignoreFailure, Map config, PipelineContext pipelineContext ) throws Exception { @@ -114,7 +117,7 @@ public FilterQueryRequestProcessor create( XContentParser parser = XContentType.JSON.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, stream) ) { - return new FilterQueryRequestProcessor(tag, description, parseInnerQueryBuilder(parser)); + return new FilterQueryRequestProcessor(tag, description, ignoreFailure, parseInnerQueryBuilder(parser)); } } } diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java index c8b3c06a71562..c410895304f76 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java @@ -19,6 +19,7 @@ import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.search.SearchHit; import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.AbstractProcessor; import org.opensearch.search.pipeline.SearchRequestProcessor; import org.opensearch.search.pipeline.SearchResponseProcessor; @@ -41,14 +42,22 @@ public class RenameFieldResponseProcessor extends AbstractProcessor implements S /** * Constructor that takes a target field to rename and the new name * - * @param tag processor tag - * @param description processor description - * @param oldField name of field to be renamed - * @param newField name of field that will replace the old field + * @param tag processor tag + * @param description processor description + * @param ignoreFailure option to ignore failure + * @param oldField name of field to be renamed + * @param newField name of field that will replace the old field * @param ignoreMissing if true, do not throw error if oldField does not exist within search response */ - public RenameFieldResponseProcessor(String tag, String description, String oldField, String newField, boolean ignoreMissing) { - super(tag, description); + public RenameFieldResponseProcessor( + String tag, + String description, + boolean ignoreFailure, + String oldField, + String newField, + boolean ignoreMissing + ) { + super(tag, description, ignoreFailure); this.oldField = oldField; this.newField = newField; this.ignoreMissing = ignoreMissing; @@ -140,13 +149,14 @@ public RenameFieldResponseProcessor create( Map> processorFactories, String tag, String description, + boolean ignoreFailure, Map config, PipelineContext pipelineContext ) throws Exception { String oldField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); String newField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field"); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", false); - return new RenameFieldResponseProcessor(tag, description, oldField, newField, ignoreMissing); + return new RenameFieldResponseProcessor(tag, description, ignoreFailure, oldField, newField, ignoreMissing); } } } diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java index 43ab3d4622d6b..e100fe0e3edce 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java @@ -25,6 +25,7 @@ import org.opensearch.script.ScriptType; import org.opensearch.script.SearchScript; import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.AbstractProcessor; import org.opensearch.search.pipeline.SearchRequestProcessor; import org.opensearch.search.pipeline.common.helpers.SearchRequestMap; @@ -54,6 +55,7 @@ public final class ScriptRequestProcessor extends AbstractProcessor implements S * * @param tag The processor's tag. * @param description The processor's description. + * @param ignoreFailure The option to ignore failure * @param script The {@link Script} to execute. * @param precompiledSearchScript The {@link Script} precompiled * @param scriptService The {@link ScriptService} used to execute the script. @@ -61,11 +63,12 @@ public final class ScriptRequestProcessor extends AbstractProcessor implements S ScriptRequestProcessor( String tag, String description, + boolean ignoreFailure, Script script, @Nullable SearchScript precompiledSearchScript, ScriptService scriptService ) { - super(tag, description); + super(tag, description, ignoreFailure); this.script = script; this.precompiledSearchScript = precompiledSearchScript; this.scriptService = scriptService; @@ -146,6 +149,7 @@ public ScriptRequestProcessor create( Map> registry, String processorTag, String description, + boolean ignoreFailure, Map config, PipelineContext pipelineContext ) throws Exception { @@ -174,7 +178,7 @@ public ScriptRequestProcessor create( } catch (ScriptException e) { throw newConfigurationException(TYPE, processorTag, null, e); } - return new ScriptRequestProcessor(processorTag, description, script, searchScript, scriptService); + return new ScriptRequestProcessor(processorTag, description, ignoreFailure, script, searchScript, scriptService); } } } diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java index ecf746af556a2..576660d05b96d 100644 --- a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessorTests.java @@ -23,7 +23,7 @@ public class FilterQueryRequestProcessorTests extends AbstractBuilderTestCase { public void testFilterQuery() throws Exception { QueryBuilder filterQuery = new TermQueryBuilder("field", "value"); - FilterQueryRequestProcessor filterQueryRequestProcessor = new FilterQueryRequestProcessor(null, null, filterQuery); + FilterQueryRequestProcessor filterQueryRequestProcessor = new FilterQueryRequestProcessor(null, null, false, filterQuery); QueryBuilder incomingQuery = new TermQueryBuilder("text", "foo"); SearchSourceBuilder source = new SearchSourceBuilder().query(incomingQuery); SearchRequest request = new SearchRequest().source(source); @@ -39,13 +39,13 @@ public void testFilterQuery() throws Exception { public void testFactory() throws Exception { FilterQueryRequestProcessor.Factory factory = new FilterQueryRequestProcessor.Factory(this.xContentRegistry()); Map configMap = new HashMap<>(Map.of("query", Map.of("term", Map.of("field", "value")))); - FilterQueryRequestProcessor processor = factory.create(Collections.emptyMap(), null, null, configMap, null); + FilterQueryRequestProcessor processor = factory.create(Collections.emptyMap(), null, null, false, configMap, null); assertEquals(new TermQueryBuilder("field", "value"), processor.filterQuery); // Missing "query" parameter: expectThrows( IllegalArgumentException.class, - () -> factory.create(Collections.emptyMap(), null, null, Collections.emptyMap(), null) + () -> factory.create(Collections.emptyMap(), null, null, false, Collections.emptyMap(), null) ); } } diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java index 7f3a2acfbdc08..b7d1510d065ae 100644 --- a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java @@ -58,6 +58,7 @@ public void testRenameResponse() throws Exception { RenameFieldResponseProcessor renameFieldResponseProcessor = new RenameFieldResponseProcessor( null, null, + false, "field 0", "new field", false @@ -74,6 +75,7 @@ public void testRenameResponseWithMapping() throws Exception { RenameFieldResponseProcessor renameFieldResponseProcessor = new RenameFieldResponseProcessor( null, null, + false, "field 0", "new field", true @@ -97,6 +99,7 @@ public void testMissingField() throws Exception { RenameFieldResponseProcessor renameFieldResponseProcessor = new RenameFieldResponseProcessor( null, null, + false, "field", "new field", false @@ -115,7 +118,7 @@ public void testFactory() throws Exception { config.put("target_field", newField); RenameFieldResponseProcessor.Factory factory = new RenameFieldResponseProcessor.Factory(); - RenameFieldResponseProcessor processor = factory.create(Collections.emptyMap(), null, null, config, null); + RenameFieldResponseProcessor processor = factory.create(Collections.emptyMap(), null, null, false, config, null); assertEquals(processor.getType(), "rename_field"); assertEquals(processor.getOldField(), oldField); assertEquals(processor.getNewField(), newField); @@ -123,7 +126,7 @@ public void testFactory() throws Exception { expectThrows( OpenSearchParseException.class, - () -> factory.create(Collections.emptyMap(), null, null, Collections.emptyMap(), null) + () -> factory.create(Collections.emptyMap(), null, null, false, Collections.emptyMap(), null) ); } } diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java index 2fb3b2345e7e2..df383e778c7ba 100644 --- a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java @@ -82,7 +82,7 @@ public void setupScripting() { } public void testScriptingWithoutPrecompiledScriptFactory() throws Exception { - ScriptRequestProcessor processor = new ScriptRequestProcessor(randomAlphaOfLength(10), null, script, null, scriptService); + ScriptRequestProcessor processor = new ScriptRequestProcessor(randomAlphaOfLength(10), null, false, script, null, scriptService); SearchRequest searchRequest = new SearchRequest(); searchRequest.source(createSearchSourceBuilder()); @@ -92,7 +92,14 @@ public void testScriptingWithoutPrecompiledScriptFactory() throws Exception { } public void testScriptingWithPrecompiledIngestScript() throws Exception { - ScriptRequestProcessor processor = new ScriptRequestProcessor(randomAlphaOfLength(10), null, script, searchScript, scriptService); + ScriptRequestProcessor processor = new ScriptRequestProcessor( + randomAlphaOfLength(10), + null, + false, + script, + searchScript, + scriptService + ); SearchRequest searchRequest = new SearchRequest(); searchRequest.source(createSearchSourceBuilder()); diff --git a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/40_rename_response.yml b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/40_rename_response.yml index 3b705f9bd5356..0528440f7584d 100644 --- a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/40_rename_response.yml +++ b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/40_rename_response.yml @@ -63,6 +63,26 @@ teardown: } - match: { acknowledged: true } + - do: + search_pipeline.put: + id: "my_pipeline_4" + body: > + { + "description": "test pipeline with ignore missing false and ignore failure true", + "response_processors": [ + { + "rename_field": + { + "field": "aa", + "target_field": "b", + "ignore_missing": false, + "ignore_failure": true + } + } + ] + } + - match: { acknowledged: true } + - do: indices.create: index: test @@ -119,8 +139,8 @@ teardown: - match: { hits.total.value: 1 } - match: {hits.hits.0._source: { "a": "foo" } } - # Pipeline with ignore_missing set to true - # Should still pass even though index does not contain field + # Pipeline with ignore_missing set to false + # Should throw illegal_argument_exception - do: catch: bad_request search: @@ -128,6 +148,15 @@ teardown: search_pipeline: "my_pipeline_3" - match: { error.type: "illegal_argument_exception" } + # Pipeline with ignore_missing set to false and ignore_failure set to true + # Should return while catching error + - do: + search: + index: test + search_pipeline: "my_pipeline_4" + - match: { hits.total.value: 1 } + - match: {hits.hits.0._source: { "a": "foo" } } + # No source, using stored_fields - do: search: diff --git a/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java index dc41d1985fe42..b212e67a59434 100644 --- a/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java @@ -67,6 +67,7 @@ public final class ConfigurationUtils { public static final String TAG_KEY = "tag"; public static final String DESCRIPTION_KEY = "description"; + public static final String IGNORE_FAILURE_KEY = "ignore_failure"; private ConfigurationUtils() {} @@ -194,7 +195,7 @@ public static String readOptionalStringOrIntProperty( return readStringOrInt(processorType, processorTag, propertyName, value); } - public static Boolean readBooleanProperty( + public static boolean readBooleanProperty( String processorType, String processorTag, Map configuration, @@ -214,7 +215,7 @@ private static Boolean readBoolean(String processorType, String processorTag, St return null; } if (value instanceof Boolean) { - return (Boolean) value; + return (boolean) value; } throw newConfigurationException( processorType, @@ -530,10 +531,11 @@ public static Processor readProcessor( ) throws Exception { String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); + boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, IGNORE_FAILURE_KEY, false); Script conditionalScript = extractConditional(config); Processor.Factory factory = processorFactories.get(type); + if (factory != null) { - boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false); List> onFailureProcessorConfigs = ConfigurationUtils.readOptionalList( null, null, diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/AbstractProcessor.java b/server/src/main/java/org/opensearch/search/pipeline/AbstractProcessor.java similarity index 60% rename from modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/AbstractProcessor.java rename to server/src/main/java/org/opensearch/search/pipeline/AbstractProcessor.java index e62497cb54db5..d197051c5952b 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/AbstractProcessor.java +++ b/server/src/main/java/org/opensearch/search/pipeline/AbstractProcessor.java @@ -6,20 +6,20 @@ * compatible open source license. */ -package org.opensearch.search.pipeline.common; - -import org.opensearch.search.pipeline.Processor; +package org.opensearch.search.pipeline; /** * Base class for common processor behavior. */ -abstract class AbstractProcessor implements Processor { +public abstract class AbstractProcessor implements Processor { private final String tag; private final String description; + private final boolean ignoreFailure; - protected AbstractProcessor(String tag, String description) { + protected AbstractProcessor(String tag, String description, boolean ignoreFailure) { this.tag = tag; this.description = description; + this.ignoreFailure = ignoreFailure; } @Override @@ -31,4 +31,9 @@ public String getTag() { public String getDescription() { return description; } + + @Override + public boolean isIgnoreFailure() { + return ignoreFailure; + } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java index 92826eee5a4f4..5ada11b59556b 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java +++ b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java @@ -8,6 +8,8 @@ package org.opensearch.search.pipeline; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchPhaseContext; import org.opensearch.action.search.SearchPhaseResults; import org.opensearch.action.search.SearchRequest; @@ -32,6 +34,9 @@ class Pipeline { public static final String REQUEST_PROCESSORS_KEY = "request_processors"; public static final String RESPONSE_PROCESSORS_KEY = "response_processors"; public static final String PHASE_PROCESSORS_KEY = "phase_results_processors"; + + private static final Logger logger = LogManager.getLogger(Pipeline.class); + private final String id; private final String description; private final Integer version; @@ -132,7 +137,18 @@ SearchRequest transformRequest(SearchRequest request) throws SearchPipelineProce request = processor.processRequest(request); } catch (Exception e) { onRequestProcessorFailed(processor); - throw e; + if (processor.isIgnoreFailure()) { + logger.warn( + "The exception from request processor [" + + processor.getType() + + "] in the search pipeline [" + + id + + "] was ignored", + e + ); + } else { + throw e; + } } finally { long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start); afterRequestProcessor(processor, took); @@ -161,7 +177,18 @@ SearchResponse transformResponse(SearchRequest request, SearchResponse response) response = processor.processResponse(request, response); } catch (Exception e) { onResponseProcessorFailed(processor); - throw e; + if (processor.isIgnoreFailure()) { + logger.warn( + "The exception from response processor [" + + processor.getType() + + "] in the search pipeline [" + + id + + "] was ignored", + e + ); + } else { + throw e; + } } finally { long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start); afterResponseProcessor(processor, took); @@ -184,12 +211,27 @@ void runSearchPhaseResultsTransformer( String currentPhase, String nextPhase ) throws SearchPipelineProcessingException { - try { for (SearchPhaseResultsProcessor searchPhaseResultsProcessor : searchPhaseResultsProcessors) { if (currentPhase.equals(searchPhaseResultsProcessor.getBeforePhase().getName()) && nextPhase.equals(searchPhaseResultsProcessor.getAfterPhase().getName())) { - searchPhaseResultsProcessor.process(searchPhaseResult, context); + try { + searchPhaseResultsProcessor.process(searchPhaseResult, context); + } catch (Exception e) { + if (searchPhaseResultsProcessor.isIgnoreFailure()) { + logger.warn( + "The exception from search phase results processor [" + + searchPhaseResultsProcessor.getType() + + "] in the search pipeline [" + + id + + "] was ignored", + e + ); + } else { + throw e; + } + } + } } } catch (RuntimeException e) { diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java index 060894a37e5ed..3e473f070d9c6 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java @@ -22,6 +22,7 @@ import java.util.function.LongSupplier; import static org.opensearch.ingest.ConfigurationUtils.TAG_KEY; +import static org.opensearch.ingest.ConfigurationUtils.IGNORE_FAILURE_KEY; import static org.opensearch.ingest.Pipeline.DESCRIPTION_KEY; import static org.opensearch.ingest.Pipeline.VERSION_KEY; @@ -150,8 +151,11 @@ private static List readProcessors( } Map config = (Map) entry.getValue(); String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); + boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, IGNORE_FAILURE_KEY, false); String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); - processors.add(processorFactories.get(type).create(processorFactories, tag, description, config, pipelineContext)); + processors.add( + processorFactories.get(type).create(processorFactories, tag, description, ignoreFailure, config, pipelineContext) + ); if (config.isEmpty() == false) { String processorName = type; if (tag != null) { diff --git a/server/src/main/java/org/opensearch/search/pipeline/Processor.java b/server/src/main/java/org/opensearch/search/pipeline/Processor.java index cc96132479c74..fb33f46acada4 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/Processor.java +++ b/server/src/main/java/org/opensearch/search/pipeline/Processor.java @@ -43,6 +43,11 @@ public interface Processor { */ String getDescription(); + /** + * Gets the setting of ignoreFailure of a processor. + */ + boolean isIgnoreFailure(); + /** * A factory that knows how to construct a processor based on a map of maps. */ @@ -63,6 +68,7 @@ T create( Map> processorFactories, String tag, String description, + boolean ignoreFailure, Map config, PipelineContext pipelineContext ) throws Exception; diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index 84f39e4bdab42..0103d4c677b00 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -78,16 +78,16 @@ public class SearchPipelineServiceTests extends OpenSearchTestCase { private static final SearchPipelinePlugin DUMMY_PLUGIN = new SearchPipelinePlugin() { @Override public Map> getRequestProcessors(Parameters parameters) { - return Map.of("foo", (factories, tag, description, config, ctx) -> null); + return Map.of("foo", (factories, tag, description, ignoreFailure, config, ctx) -> null); } public Map> getResponseProcessors(Parameters parameters) { - return Map.of("bar", (factories, tag, description, config, ctx) -> null); + return Map.of("bar", (factories, tag, description, ignoreFailure, config, ctx) -> null); } @Override public Map> getSearchPhaseResultsProcessors(Parameters parameters) { - return Map.of("zoe", (factories, tag, description, config, ctx) -> null); + return Map.of("zoe", (factories, tag, description, ignoreFailure, config, ctx) -> null); } }; @@ -208,39 +208,15 @@ public void testResolveIndexDefaultPipeline() throws Exception { assertEquals(5, pipelinedRequest.source().size()); } - private static abstract class FakeProcessor implements Processor { - private final String type; - private final String tag; - private final String description; - - protected FakeProcessor(String type, String tag, String description) { - this.type = type; - this.tag = tag; - this.description = description; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getTag() { - return tag; - } - - @Override - public String getDescription() { - return description; - } - } - - private static class FakeRequestProcessor extends FakeProcessor implements SearchRequestProcessor { + private static class FakeRequestProcessor extends AbstractProcessor implements SearchRequestProcessor { private final Consumer executor; - public FakeRequestProcessor(String type, String tag, String description, Consumer executor) { - super(type, tag, description); + private final String type; + + public FakeRequestProcessor(String type, String tag, String description, boolean ignoreFailure, Consumer executor) { + super(tag, description, ignoreFailure); this.executor = executor; + this.type = type; } @Override @@ -248,14 +224,30 @@ public SearchRequest processRequest(SearchRequest request) throws Exception { executor.accept(request); return request; } + + /* + * Gets the type of processor + */ + @Override + public String getType() { + return this.type; + } } - private static class FakeResponseProcessor extends FakeProcessor implements SearchResponseProcessor { + private static class FakeResponseProcessor extends AbstractProcessor implements SearchResponseProcessor { private final Consumer executor; + private String type; - public FakeResponseProcessor(String type, String tag, String description, Consumer executor) { - super(type, tag, description); + public FakeResponseProcessor( + String type, + String tag, + String description, + boolean ignoreFailure, + Consumer executor + ) { + super(tag, description, ignoreFailure); this.executor = executor; + this.type = type; } @Override @@ -263,19 +255,31 @@ public SearchResponse processResponse(SearchRequest request, SearchResponse resp executor.accept(response); return response; } + + /** + * Gets the type of processor + */ + @Override + public String getType() { + return this.type; + } } - private static class FakeSearchPhaseResultsProcessor extends FakeProcessor implements SearchPhaseResultsProcessor { + private static class FakeSearchPhaseResultsProcessor extends AbstractProcessor implements SearchPhaseResultsProcessor { private Consumer querySearchResultConsumer; + private String type; + public FakeSearchPhaseResultsProcessor( String type, String tag, String description, + boolean ignoreFailure, Consumer querySearchResultConsumer ) { - super(type, tag, description); + super(tag, description, ignoreFailure); this.querySearchResultConsumer = querySearchResultConsumer; + this.type = type; } @Override @@ -297,30 +301,45 @@ public SearchPhaseName getBeforePhase() { public SearchPhaseName getAfterPhase() { return SearchPhaseName.FETCH; } + + /** + * Gets the type of processor + */ + @Override + public String getType() { + return this.type; + } } private SearchPipelineService createWithProcessors() { Map> requestProcessors = new HashMap<>(); - requestProcessors.put("scale_request_size", (processorFactories, tag, description, config, ctx) -> { + requestProcessors.put("scale_request_size", (processorFactories, tag, description, ignoreFailure, config, ctx) -> { float scale = ((Number) config.remove("scale")).floatValue(); return new FakeRequestProcessor( "scale_request_size", tag, description, + ignoreFailure, req -> req.source().size((int) (req.source().size() * scale)) ); }); Map> responseProcessors = new HashMap<>(); - responseProcessors.put("fixed_score", (processorFactories, tag, description, config, ctx) -> { + responseProcessors.put("fixed_score", (processorFactories, tag, description, ignoreFailure, config, ctx) -> { float score = ((Number) config.remove("score")).floatValue(); - return new FakeResponseProcessor("fixed_score", tag, description, rsp -> rsp.getHits().forEach(h -> h.score(score))); + return new FakeResponseProcessor( + "fixed_score", + tag, + description, + ignoreFailure, + rsp -> rsp.getHits().forEach(h -> h.score(score)) + ); }); Map> searchPhaseProcessors = new HashMap<>(); - searchPhaseProcessors.put("max_score", (processorFactories, tag, description, config, context) -> { + searchPhaseProcessors.put("max_score", (processorFactories, tag, description, ignoreFailure, config, context) -> { final float finalScore = config.containsKey("score") ? ((Number) config.remove("score")).floatValue() : 100f; final Consumer querySearchResultConsumer = (result) -> result.queryResult().topDocs().maxScore = finalScore; - return new FakeSearchPhaseResultsProcessor("max_score", tag, description, querySearchResultConsumer); + return new FakeSearchPhaseResultsProcessor("max_score", tag, description, ignoreFailure, querySearchResultConsumer); }); return createWithProcessors(requestProcessors, responseProcessors, searchPhaseProcessors); @@ -893,7 +912,7 @@ public void testInfo() { } public void testExceptionOnPipelineCreation() { - Map> badFactory = Map.of("bad_factory", (pf, t, f, c, ctx) -> { + Map> badFactory = Map.of("bad_factory", (pf, t, i, f, c, ctx) -> { throw new RuntimeException(); }); SearchPipelineService searchPipelineService = createWithProcessors(badFactory, Collections.emptyMap(), Collections.emptyMap()); @@ -910,12 +929,12 @@ public void testExceptionOnPipelineCreation() { } public void testExceptionOnRequestProcessing() { - SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", null, null, r -> { + SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", null, null, false, r -> { throw new RuntimeException(); }); Map> throwingRequestProcessorFactory = Map.of( "throwing_request", - (pf, t, f, c, ctx) -> throwingRequestProcessor + (pf, t, i, f, c, ctx) -> throwingRequestProcessor ); SearchPipelineService searchPipelineService = createWithProcessors( @@ -935,12 +954,12 @@ public void testExceptionOnRequestProcessing() { } public void testExceptionOnResponseProcessing() throws Exception { - SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", null, null, r -> { + SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", null, null, false, r -> { throw new RuntimeException(); }); Map> throwingResponseProcessorFactory = Map.of( "throwing_response", - (pf, t, f, c, ctx) -> throwingResponseProcessor + (pf, t, i, f, c, ctx) -> throwingResponseProcessor ); SearchPipelineService searchPipelineService = createWithProcessors( @@ -962,61 +981,103 @@ public void testExceptionOnResponseProcessing() throws Exception { expectThrows(SearchPipelineProcessingException.class, () -> pipelinedRequest.transformResponse(response)); } + public void testCatchExceptionOnRequestProcessing() throws IllegalAccessException { + SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", null, null, true, r -> { + throw new RuntimeException(); + }); + Map> throwingRequestProcessorFactory = Map.of( + "throwing_request", + (pf, t, i, f, c, ctx) -> throwingRequestProcessor + ); + + SearchPipelineService searchPipelineService = createWithProcessors( + throwingRequestProcessorFactory, + Collections.emptyMap(), + Collections.emptyMap() + ); + + Map pipelineSourceMap = new HashMap<>(); + pipelineSourceMap.put(Pipeline.REQUEST_PROCESSORS_KEY, List.of(Map.of("throwing_request", Collections.emptyMap()))); + + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().searchPipelineSource(pipelineSourceMap); + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + + // Caught Exception thrown when processing the request and produced warn level logging message + try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(Pipeline.class))) { + mockAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "test1", + Pipeline.class.getCanonicalName(), + Level.WARN, + "The exception from request processor [throwing_request] in the search pipeline [_ad_hoc_pipeline] was ignored" + ) + ); + PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest); + mockAppender.assertAllExpectationsMatched(); + } + } + + public void testCatchExceptionOnResponseProcessing() throws Exception { + SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", null, null, true, r -> { + throw new RuntimeException(); + }); + Map> throwingResponseProcessorFactory = Map.of( + "throwing_response", + (pf, t, i, f, c, ctx) -> throwingResponseProcessor + ); + + SearchPipelineService searchPipelineService = createWithProcessors( + Collections.emptyMap(), + throwingResponseProcessorFactory, + Collections.emptyMap() + ); + + Map pipelineSourceMap = new HashMap<>(); + pipelineSourceMap.put(Pipeline.RESPONSE_PROCESSORS_KEY, List.of(Map.of("throwing_response", Collections.emptyMap()))); + + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).searchPipelineSource(pipelineSourceMap); + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + + PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest); + + SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null); + + // Caught Exception thrown when processing response and produced warn level logging message + try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(Pipeline.class))) { + mockAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "test1", + Pipeline.class.getCanonicalName(), + Level.WARN, + "The exception from response processor [throwing_response] in the search pipeline [_ad_hoc_pipeline] was ignored" + ) + ); + pipelinedRequest.transformResponse(response); + mockAppender.assertAllExpectationsMatched(); + } + } + public void testStats() throws Exception { - SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", "1", null, r -> { + SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", "1", null, false, r -> { throw new RuntimeException(); }); Map> requestProcessors = Map.of( "successful_request", - (pf, t, f, c, ctx) -> new FakeRequestProcessor("successful_request", "2", null, r -> {}), + (pf, t, i, f, c, ctx) -> new FakeRequestProcessor("successful_request", "2", null, false, r -> {}), "throwing_request", - (pf, t, f, c, ctx) -> throwingRequestProcessor + (pf, t, i, f, c, ctx) -> throwingRequestProcessor ); - SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", "3", null, r -> { + SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", "3", null, false, r -> { throw new RuntimeException(); }); Map> responseProcessors = Map.of( "successful_response", - (pf, t, f, c, ctx) -> new FakeResponseProcessor("successful_response", "4", null, r -> {}), + (pf, t, i, f, c, ctx) -> new FakeResponseProcessor("successful_response", "4", null, false, r -> {}), "throwing_response", - (pf, t, f, c, ctx) -> throwingResponseProcessor + (pf, t, i, f, c, ctx) -> throwingResponseProcessor ); - SearchPipelineService searchPipelineService = createWithProcessors(requestProcessors, responseProcessors, Collections.emptyMap()); - SearchPipelineMetadata metadata = new SearchPipelineMetadata( - Map.of( - "good_response_pipeline", - new PipelineConfiguration( - "good_response_pipeline", - new BytesArray("{\"response_processors\" : [ { \"successful_response\": {} } ] }"), - XContentType.JSON - ), - "bad_response_pipeline", - new PipelineConfiguration( - "bad_response_pipeline", - new BytesArray("{\"response_processors\" : [ { \"throwing_response\": {} } ] }"), - XContentType.JSON - ), - "good_request_pipeline", - new PipelineConfiguration( - "good_request_pipeline", - new BytesArray("{\"request_processors\" : [ { \"successful_request\": {} } ] }"), - XContentType.JSON - ), - "bad_request_pipeline", - new PipelineConfiguration( - "bad_request_pipeline", - new BytesArray("{\"request_processors\" : [ { \"throwing_request\": {} } ] }"), - XContentType.JSON - ) - ) - ); - ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); - ClusterState previousState = clusterState; - clusterState = ClusterState.builder(clusterState) - .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata)) - .build(); - searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState)); + SearchPipelineService searchPipelineService = getSearchPipelineService(requestProcessors, responseProcessors); SearchRequest request = new SearchRequest(); SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null); @@ -1079,6 +1140,142 @@ public void testStats() throws Exception { } } + public void testStatsEnabledIgnoreFailure() throws Exception { + SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", "1", null, true, r -> { + throw new RuntimeException(); + }); + Map> requestProcessorsEnableIgnoreFailure = Map.of( + "successful_request", + (pf, t, i, f, c, ctx) -> new FakeRequestProcessor("successful_request", "2", null, true, r -> {}), + "throwing_request", + (pf, t, i, f, c, ctx) -> throwingRequestProcessor + ); + SearchResponseProcessor throwingResponseProcessor = new FakeResponseProcessor("throwing_response", "3", null, true, r -> { + throw new RuntimeException(); + }); + Map> responseProcessorsEnableIgnoreFailure = Map.of( + "successful_response", + (pf, t, i, f, c, ctx) -> new FakeResponseProcessor("successful_response", "4", null, true, r -> {}), + "throwing_response", + (pf, t, i, f, c, ctx) -> throwingResponseProcessor + ); + + SearchPipelineService searchPipelineService = getSearchPipelineService( + requestProcessorsEnableIgnoreFailure, + responseProcessorsEnableIgnoreFailure + ); + + SearchRequest request = new SearchRequest(); + SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null); + + searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")).transformResponse(response); + // Caught Exception here + searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")).transformResponse(response); + searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")).transformResponse(response); + // Caught Exception here + searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")).transformResponse(response); + + // when ignoreFailure enabled, the search pipelines will all succeed. + SearchPipelineStats stats = searchPipelineService.stats(); + assertPipelineStats(stats.getTotalRequestStats(), 2, 0); + assertPipelineStats(stats.getTotalResponseStats(), 2, 0); + + for (SearchPipelineStats.PerPipelineStats perPipelineStats : stats.getPipelineStats()) { + SearchPipelineStats.PipelineDetailStats detailStats = stats.getPerPipelineProcessorStats() + .get(perPipelineStats.getPipelineId()); + switch (perPipelineStats.getPipelineId()) { + case "good_request_pipeline": + assertPipelineStats(perPipelineStats.getRequestStats(), 1, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 0, 0); + assertEquals(1, detailStats.requestProcessorStats().size()); + assertEquals(0, detailStats.responseProcessorStats().size()); + assertEquals("successful_request:2", detailStats.requestProcessorStats().get(0).getProcessorName()); + assertEquals("successful_request", detailStats.requestProcessorStats().get(0).getProcessorType()); + assertPipelineStats(detailStats.requestProcessorStats().get(0).getStats(), 1, 0); + break; + case "bad_request_pipeline": + // pipeline succeed when ignore failure is true + assertPipelineStats(perPipelineStats.getRequestStats(), 1, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 0, 0); + assertEquals(1, detailStats.requestProcessorStats().size()); + assertEquals(0, detailStats.responseProcessorStats().size()); + assertEquals("throwing_request:1", detailStats.requestProcessorStats().get(0).getProcessorName()); + assertEquals("throwing_request", detailStats.requestProcessorStats().get(0).getProcessorType()); + // processor stats got 1 count and 1 failed + assertPipelineStats(detailStats.requestProcessorStats().get(0).getStats(), 1, 1); + break; + case "good_response_pipeline": + assertPipelineStats(perPipelineStats.getRequestStats(), 0, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 1, 0); + assertEquals(0, detailStats.requestProcessorStats().size()); + assertEquals(1, detailStats.responseProcessorStats().size()); + assertEquals("successful_response:4", detailStats.responseProcessorStats().get(0).getProcessorName()); + assertEquals("successful_response", detailStats.responseProcessorStats().get(0).getProcessorType()); + assertPipelineStats(detailStats.responseProcessorStats().get(0).getStats(), 1, 0); + break; + case "bad_response_pipeline": + // pipeline succeed when ignore failure is true + assertPipelineStats(perPipelineStats.getRequestStats(), 0, 0); + assertPipelineStats(perPipelineStats.getResponseStats(), 1, 0); + assertEquals(0, detailStats.requestProcessorStats().size()); + assertEquals(1, detailStats.responseProcessorStats().size()); + assertEquals("throwing_response:3", detailStats.responseProcessorStats().get(0).getProcessorName()); + assertEquals("throwing_response", detailStats.responseProcessorStats().get(0).getProcessorType()); + // processor stats got 1 count and 1 failed + assertPipelineStats(detailStats.responseProcessorStats().get(0).getStats(), 1, 1); + break; + } + } + + } + + private SearchPipelineService getSearchPipelineService( + Map> requestProcessorsEnableIgnoreFailure, + Map> responseProcessorsEnableIgnoreFailure + ) { + SearchPipelineService searchPipelineService = createWithProcessors( + requestProcessorsEnableIgnoreFailure, + responseProcessorsEnableIgnoreFailure, + Collections.emptyMap() + ); + + SearchPipelineMetadata metadata = new SearchPipelineMetadata( + Map.of( + "good_response_pipeline", + new PipelineConfiguration( + "good_response_pipeline", + new BytesArray("{\"response_processors\" : [ { \"successful_response\": {} } ] }"), + XContentType.JSON + ), + "bad_response_pipeline", + new PipelineConfiguration( + "bad_response_pipeline", + new BytesArray("{\"response_processors\" : [ { \"throwing_response\": {} } ] }"), + XContentType.JSON + ), + "good_request_pipeline", + new PipelineConfiguration( + "good_request_pipeline", + new BytesArray("{\"request_processors\" : [ { \"successful_request\": {} } ] }"), + XContentType.JSON + ), + "bad_request_pipeline", + new PipelineConfiguration( + "bad_request_pipeline", + new BytesArray("{\"request_processors\" : [ { \"throwing_request\": {} } ] }"), + XContentType.JSON + ) + ) + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousState = clusterState; + clusterState = ClusterState.builder(clusterState) + .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata)) + .build(); + searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState)); + return searchPipelineService; + } + private static void assertPipelineStats(OperationStats stats, long count, long failed) { assertEquals(stats.getCount(), count); assertEquals(stats.getFailedCount(), failed); @@ -1086,11 +1283,11 @@ private static void assertPipelineStats(OperationStats stats, long count, long f public void testAdHocRejectingProcessor() { String processorType = "ad_hoc_rejecting"; - Map> requestProcessorFactories = Map.of(processorType, (pf, t, d, c, ctx) -> { + Map> requestProcessorFactories = Map.of(processorType, (pf, t, d, i, c, ctx) -> { if (ctx.getPipelineSource() == Processor.PipelineSource.SEARCH_REQUEST) { throw new IllegalArgumentException(processorType + " cannot be created as part of a pipeline defined in a search request"); } - return new FakeRequestProcessor(processorType, t, d, r -> {}); + return new FakeRequestProcessor(processorType, t, d, i, r -> {}); }); SearchPipelineService searchPipelineService = createWithProcessors( From 051550112c7ae7ae1e81f9da3eff10c73407cf38 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 12 Jul 2023 03:48:59 +0800 Subject: [PATCH 107/186] Flip SEARCH_PIPELINE feature flag for search pipeline GA (#8613) Signed-off-by: Louis Chu --- CHANGELOG.md | 1 + .../src/main/java/org/opensearch/common/util/FeatureFlags.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0de0d823aa764..ae6752400b90b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Search pipelines] Add Global Ignore_failure options for Processors ([#8373](https://github.com/opensearch-project/OpenSearch/pull/8373)) - Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) - Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) +- Introduce full support for Search Pipeline ([#8613](https://github.com/opensearch-project/OpenSearch/pull/8613)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 2b4dab616d00f..915f231a1ea49 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -106,7 +106,7 @@ public static boolean isEnabled(String featureFlagName) { public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); - public static final Setting SEARCH_PIPELINE_SETTING = Setting.boolSetting(SEARCH_PIPELINE, false, Property.NodeScope); + public static final Setting SEARCH_PIPELINE_SETTING = Setting.boolSetting(SEARCH_PIPELINE, true, Property.NodeScope); public static final Setting IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope); From 8d2ec7673711be0f075f283e3c4177094d31d450 Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Tue, 11 Jul 2023 15:10:29 -0500 Subject: [PATCH 108/186] [Refactor] HPPC IntArrayList and ObjectArrayList to j.u.List (#8602) Refactors usage of obsolete HPPC IntArrayList and ObjectArrayList to java.util.List. Signed-off-by: Nicholas Walter Knize --- .../action/get/MultiGetShardRequest.java | 7 +++---- .../action/get/MultiGetShardResponse.java | 7 +++---- .../action/search/FetchSearchPhase.java | 7 +++---- .../action/search/SearchPhaseController.java | 9 ++++----- .../SearchScrollQueryThenFetchAsyncAction.java | 6 +++--- .../MultiTermVectorsShardRequest.java | 7 +++---- .../MultiTermVectorsShardResponse.java | 7 +++---- .../opensearch/common/transport/PortsRange.java | 17 +++++++---------- .../search/fetch/ShardFetchRequest.java | 6 +++--- .../search/fetch/ShardFetchSearchRequest.java | 4 ++-- .../opensearch/search/SearchServiceTests.java | 3 +-- .../java/org/opensearch/test/TestCluster.java | 10 ++++++---- 12 files changed, 41 insertions(+), 49 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java b/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java index 22d710c38a8c9..23ebf91fe2300 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java @@ -32,7 +32,6 @@ package org.opensearch.action.get; -import com.carrotsearch.hppc.IntArrayList; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.common.io.stream.StreamInput; @@ -54,13 +53,13 @@ public class MultiGetShardRequest extends SingleShardRequest locations; List items; MultiGetShardRequest(StreamInput in) throws IOException { super(in); int size = in.readVInt(); - locations = new IntArrayList(size); + locations = new ArrayList<>(size); items = new ArrayList<>(size); for (int i = 0; i < size; i++) { @@ -76,7 +75,7 @@ public class MultiGetShardRequest extends SingleShardRequest(); items = new ArrayList<>(); preference = multiGetRequest.preference; realtime = multiGetRequest.realtime; diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java b/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java index 069f9875185f8..93adaa66f437d 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java @@ -32,7 +32,6 @@ package org.opensearch.action.get; -import com.carrotsearch.hppc.IntArrayList; import org.opensearch.action.ActionResponse; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -48,12 +47,12 @@ */ public class MultiGetShardResponse extends ActionResponse { - final IntArrayList locations; + final List locations; final List responses; final List failures; MultiGetShardResponse() { - locations = new IntArrayList(); + locations = new ArrayList<>(); responses = new ArrayList<>(); failures = new ArrayList<>(); } @@ -61,7 +60,7 @@ public class MultiGetShardResponse extends ActionResponse { MultiGetShardResponse(StreamInput in) throws IOException { super(in); int size = in.readVInt(); - locations = new IntArrayList(size); + locations = new ArrayList<>(size); responses = new ArrayList<>(size); failures = new ArrayList<>(size); for (int i = 0; i < size; i++) { diff --git a/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java index 85a3d140977bb..ebb2f33f8f37d 100644 --- a/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java @@ -31,7 +31,6 @@ package org.opensearch.action.search; -import com.carrotsearch.hppc.IntArrayList; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.search.ScoreDoc; @@ -151,7 +150,7 @@ private void innerRun() throws Exception { finishPhase.run(); } else { ScoreDoc[] scoreDocs = reducedQueryPhase.sortedTopDocs.scoreDocs; - final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(numShards, scoreDocs); + final List[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(numShards, scoreDocs); // no docs to fetch -- sidestep everything and return if (scoreDocs.length == 0) { // we have to release contexts here to free up resources @@ -168,7 +167,7 @@ private void innerRun() throws Exception { context ); for (int i = 0; i < docIdsToLoad.length; i++) { - IntArrayList entry = docIdsToLoad[i]; + List entry = docIdsToLoad[i]; SearchPhaseResult queryResult = queryResults.get(i); if (entry == null) { // no results for this shard ID if (queryResult != null) { @@ -205,7 +204,7 @@ private void innerRun() throws Exception { protected ShardFetchSearchRequest createFetchRequest( ShardSearchContextId contextId, int index, - IntArrayList entry, + List entry, ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices, ShardSearchRequest shardSearchRequest, diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index a4984db7c4095..133f0a6be4628 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -32,7 +32,6 @@ package org.opensearch.action.search; -import com.carrotsearch.hppc.IntArrayList; import com.carrotsearch.hppc.ObjectObjectHashMap; import org.apache.lucene.index.Term; @@ -277,12 +276,12 @@ public ScoreDoc[] getLastEmittedDocPerShard(ReducedQueryPhase reducedQueryPhase, /** * Builds an array, with potential null elements, with docs to load. */ - public IntArrayList[] fillDocIdsToLoad(int numShards, ScoreDoc[] shardDocs) { - IntArrayList[] docIdsToLoad = new IntArrayList[numShards]; + public List[] fillDocIdsToLoad(int numShards, ScoreDoc[] shardDocs) { + final List[] docIdsToLoad = (List[]) new ArrayList[numShards]; for (ScoreDoc shardDoc : shardDocs) { - IntArrayList shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex]; + List shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex]; if (shardDocIdsToLoad == null) { - shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex] = new IntArrayList(); + shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex] = new ArrayList<>(); } shardDocIdsToLoad.add(shardDoc.doc); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java index 9c0721ef63ea6..1fa6460a212f7 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/SearchScrollQueryThenFetchAsyncAction.java @@ -32,7 +32,6 @@ package org.opensearch.action.search; -import com.carrotsearch.hppc.IntArrayList; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.ScoreDoc; import org.opensearch.action.ActionListener; @@ -48,6 +47,7 @@ import org.opensearch.search.query.ScrollQuerySearchResult; import org.opensearch.transport.Transport; +import java.util.List; import java.util.function.BiFunction; /** @@ -104,7 +104,7 @@ public void run() { return; } - final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(queryResults.length(), scoreDocs); + final List[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(queryResults.length(), scoreDocs); final ScoreDoc[] lastEmittedDocPerShard = searchPhaseController.getLastEmittedDocPerShard( reducedQueryPhase, queryResults.length() @@ -112,7 +112,7 @@ public void run() { final CountDown counter = new CountDown(docIdsToLoad.length); for (int i = 0; i < docIdsToLoad.length; i++) { final int index = i; - final IntArrayList docIds = docIdsToLoad[index]; + final List docIds = docIdsToLoad[index]; if (docIds != null) { final QuerySearchResult querySearchResult = queryResults.get(index); ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[index]; diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java index e936dd3a658a3..bf2d4c676f409 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java @@ -32,7 +32,6 @@ package org.opensearch.action.termvectors; -import com.carrotsearch.hppc.IntArrayList; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.common.io.stream.StreamInput; @@ -52,13 +51,13 @@ public class MultiTermVectorsShardRequest extends SingleShardRequest locations; List requests; MultiTermVectorsShardRequest(StreamInput in) throws IOException { super(in); int size = in.readVInt(); - locations = new IntArrayList(size); + locations = new ArrayList<>(size); requests = new ArrayList<>(size); for (int i = 0; i < size; i++) { locations.add(in.readVInt()); @@ -71,7 +70,7 @@ public class MultiTermVectorsShardRequest extends SingleShardRequest(); requests = new ArrayList<>(); } diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java index c819c591468ea..806510f58f03a 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java @@ -32,7 +32,6 @@ package org.opensearch.action.termvectors; -import com.carrotsearch.hppc.IntArrayList; import org.opensearch.action.ActionResponse; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -48,12 +47,12 @@ */ public class MultiTermVectorsShardResponse extends ActionResponse { - final IntArrayList locations; + final List locations; final List responses; final List failures; MultiTermVectorsShardResponse() { - locations = new IntArrayList(); + locations = new ArrayList<>(); responses = new ArrayList<>(); failures = new ArrayList<>(); } @@ -61,7 +60,7 @@ public class MultiTermVectorsShardResponse extends ActionResponse { MultiTermVectorsShardResponse(StreamInput in) throws IOException { super(in); int size = in.readVInt(); - locations = new IntArrayList(size); + locations = new ArrayList<>(size); responses = new ArrayList<>(size); failures = new ArrayList<>(size); for (int i = 0; i < size; i++) { diff --git a/server/src/main/java/org/opensearch/common/transport/PortsRange.java b/server/src/main/java/org/opensearch/common/transport/PortsRange.java index 30ab5e355f090..daf3e00c062fa 100644 --- a/server/src/main/java/org/opensearch/common/transport/PortsRange.java +++ b/server/src/main/java/org/opensearch/common/transport/PortsRange.java @@ -32,8 +32,8 @@ package org.opensearch.common.transport; -import com.carrotsearch.hppc.IntArrayList; - +import java.util.ArrayList; +import java.util.List; import java.util.StringTokenizer; /** @@ -54,15 +54,12 @@ public String getPortRangeString() { } public int[] ports() throws NumberFormatException { - final IntArrayList ports = new IntArrayList(); - iterate(new PortCallback() { - @Override - public boolean onPortNumber(int portNumber) { - ports.add(portNumber); - return false; - } + final List ports = new ArrayList<>(); + iterate(portNumber -> { + ports.add(portNumber); + return false; }); - return ports.toArray(); + return ports.stream().mapToInt(Integer::intValue).toArray(); } public boolean iterate(PortCallback callback) throws NumberFormatException { diff --git a/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java index 8bccc4425b46b..5c2a7d5bf974e 100644 --- a/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java +++ b/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java @@ -32,7 +32,6 @@ package org.opensearch.search.fetch; -import com.carrotsearch.hppc.IntArrayList; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.opensearch.action.search.SearchShardTask; @@ -49,6 +48,7 @@ import org.opensearch.transport.TransportRequest; import java.io.IOException; +import java.util.Collection; import java.util.Map; /** @@ -67,9 +67,9 @@ public class ShardFetchRequest extends TransportRequest { private ScoreDoc lastEmittedDoc; - public ShardFetchRequest(ShardSearchContextId contextId, IntArrayList list, ScoreDoc lastEmittedDoc) { + public ShardFetchRequest(ShardSearchContextId contextId, Collection list, ScoreDoc lastEmittedDoc) { this.contextId = contextId; - this.docIds = list.buffer; + this.docIds = list.stream().mapToInt(Integer::intValue).toArray(); this.size = list.size(); this.lastEmittedDoc = lastEmittedDoc; } diff --git a/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java index 2c0676ead333a..32b2f71522856 100644 --- a/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java @@ -32,7 +32,6 @@ package org.opensearch.search.fetch; -import com.carrotsearch.hppc.IntArrayList; import org.apache.lucene.search.ScoreDoc; import org.opensearch.action.IndicesRequest; import org.opensearch.action.OriginalIndices; @@ -45,6 +44,7 @@ import org.opensearch.search.internal.ShardSearchRequest; import java.io.IOException; +import java.util.List; /** * Shard level fetch request used with search. Holds indices taken from the original search request @@ -63,7 +63,7 @@ public ShardFetchSearchRequest( OriginalIndices originalIndices, ShardSearchContextId id, ShardSearchRequest shardSearchRequest, - IntArrayList list, + List list, ScoreDoc lastEmittedDoc, RescoreDocIds rescoreDocIds, AggregatedDfs aggregatedDfs diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java index 74ef289c4b75f..74d1df94214ee 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java @@ -31,7 +31,6 @@ package org.opensearch.search; -import com.carrotsearch.hppc.IntArrayList; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.LeafReader; @@ -360,7 +359,7 @@ public void onFailure(Exception e) { result ); SearchPhaseResult searchPhaseResult = result.get(); - IntArrayList intCursors = new IntArrayList(1); + List intCursors = new ArrayList(1); intCursors.add(0); ShardFetchRequest req = new ShardFetchRequest(searchPhaseResult.getContextId(), intCursors, null/* not a scroll */); PlainActionFuture listener = new PlainActionFuture<>(); diff --git a/test/framework/src/main/java/org/opensearch/test/TestCluster.java b/test/framework/src/main/java/org/opensearch/test/TestCluster.java index 478b692fb06ef..3dcaaefea61f9 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/TestCluster.java @@ -32,8 +32,6 @@ package org.opensearch.test; -import com.carrotsearch.hppc.ObjectArrayList; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; @@ -53,6 +51,8 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.Set; @@ -189,12 +189,14 @@ public void wipeIndices(String... indices) { // which is the case in the CloseIndexDisableCloseAllTests if ("_all".equals(indices[0])) { ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet(); - ObjectArrayList concreteIndices = new ObjectArrayList<>(); + List concreteIndices = new ArrayList<>(); for (IndexMetadata indexMetadata : clusterStateResponse.getState().metadata()) { concreteIndices.add(indexMetadata.getIndex().getName()); } if (!concreteIndices.isEmpty()) { - OpenSearchAssertions.assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(String.class))); + OpenSearchAssertions.assertAcked( + client().admin().indices().prepareDelete(concreteIndices.toArray(new String[concreteIndices.size()])) + ); } } } From ecd3f0be54c3f731a814bd2bae3b30c706a8af7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:30:37 -0700 Subject: [PATCH 109/186] Bump net.minidev:json-smart from 2.4.11 to 2.5.0 in /plugins/repository-hdfs (#8575) * Bump net.minidev:json-smart in /plugins/repository-hdfs Bumps [net.minidev:json-smart](https://github.com/netplex/json-smart-v2) from 2.4.11 to 2.5.0. - [Release notes](https://github.com/netplex/json-smart-v2/releases) - [Commits](https://github.com/netplex/json-smart-v2/compare/2.4.11...2.5.0) --- updated-dependencies: - dependency-name: net.minidev:json-smart dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] Signed-off-by: owaiskazi19 --------- Signed-off-by: dependabot[bot] Signed-off-by: owaiskazi19 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/json-smart-2.4.11.jar.sha1 | 1 - plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/json-smart-2.4.11.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index ae6752400b90b..7e4c7f47e950a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,7 +141,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) - Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) -- Bump `net.minidev:json-smart` from 2.4.11 to 2.5.0 ([#8576](https://github.com/opensearch-project/OpenSearch/pull/8576)) +- Bump `net.minidev:json-smart` from 2.4.11 to 2.5.0 (#8575, #8576) - Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) - Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) - Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 3c83e535a91d8..0b13e18ba3f4d 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -81,7 +81,7 @@ dependencies { api 'javax.servlet:servlet-api:2.5' api "org.slf4j:slf4j-api:${versions.slf4j}" api "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" - api 'net.minidev:json-smart:2.4.11' + api 'net.minidev:json-smart:2.5.0' api "io.netty:netty-all:${versions.netty}" implementation "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" implementation 'org.codehaus.woodstox:stax2-api:4.2.1' diff --git a/plugins/repository-hdfs/licenses/json-smart-2.4.11.jar.sha1 b/plugins/repository-hdfs/licenses/json-smart-2.4.11.jar.sha1 deleted file mode 100644 index 04627ab2baace..0000000000000 --- a/plugins/repository-hdfs/licenses/json-smart-2.4.11.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cc5888f14a5768f254b97bafe8b9fd29b31e872e \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1 b/plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1 new file mode 100644 index 0000000000000..3ec055efa1255 --- /dev/null +++ b/plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1 @@ -0,0 +1 @@ +57a64f421b472849c40e77d2e7cce3a141b41e99 \ No newline at end of file From 89533c80b7d79eec41f6f478196886ff3bf60193 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:35:28 -0400 Subject: [PATCH 110/186] Manually update com.google.jimfs:jimfs from 1.2 to 1.3.0 (#8585) * update Jimfs Signed-off-by: Stephen Crawford * Add PR number Signed-off-by: Stephen Crawford * bump in keystore too Signed-off-by: Stephen Crawford * bump in upgrade-cli Signed-off-by: Stephen Crawford * Bump in evil tests Signed-off-by: Stephen Crawford * Specify extended path Signed-off-by: Stephen Crawford * Fix third party audit Signed-off-by: Stephen Crawford * trivial change to trigger retry Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> --- CHANGELOG.md | 1 + distribution/tools/plugin-cli/build.gradle | 2 +- qa/evil-tests/build.gradle | 3 +-- .../src/main/java/org/opensearch/identity/IdentityService.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4c7f47e950a..da9460f820f96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) - Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) - Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) +- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 ([#8585](https://github.com/opensearch-project/OpenSearch/pull/8585)) - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) ### Changed diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 7a300c17c8189..5103999428814 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -40,7 +40,7 @@ dependencies { api "org.bouncycastle:bcpg-fips:1.0.7.1" api "org.bouncycastle:bc-fips:1.0.2.3" testImplementation project(":test:framework") - testImplementation 'com.google.jimfs:jimfs:1.2' + testImplementation 'com.google.jimfs:jimfs:1.3.0' testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index 3d97aab077251..91cb4c7151a69 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -40,7 +40,7 @@ apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.standalone-test' dependencies { - testImplementation 'com.google.jimfs:jimfs:1.2' + testImplementation 'com.google.jimfs:jimfs:1.3.0' } // TODO: give each evil test its own fresh JVM for more isolation. @@ -62,7 +62,6 @@ thirdPartyAudit { 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', - 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', 'com.google.common.hash.Striped64', 'com.google.common.hash.Striped64$1', 'com.google.common.hash.Striped64$Cell', diff --git a/server/src/main/java/org/opensearch/identity/IdentityService.java b/server/src/main/java/org/opensearch/identity/IdentityService.java index 1ce107f743efc..54a11c8b31fb3 100644 --- a/server/src/main/java/org/opensearch/identity/IdentityService.java +++ b/server/src/main/java/org/opensearch/identity/IdentityService.java @@ -16,7 +16,7 @@ import org.opensearch.plugins.IdentityPlugin; /** - * Identity and access control for OpenSearch. + * Identity and access control for OpenSearch * * @opensearch.experimental * */ From 32e72619251957cc3e045b29d13f4e3c190a5008 Mon Sep 17 00:00:00 2001 From: Ankit Kala Date: Wed, 12 Jul 2023 02:59:34 +0530 Subject: [PATCH 111/186] Start replication checkpointTimers on primary before segments upload to remote store. (#8221) * Start replication timer before segments upload. Signed-off-by: Ankit Kala * Addressed comments Signed-off-by: Ankit Kala --------- Signed-off-by: Ankit Kala --- CHANGELOG.md | 1 + .../java/org/opensearch/index/shard/IndexShard.java | 7 ++++--- .../index/shard/RemoteStoreRefreshListener.java | 3 ++- .../org/opensearch/index/shard/IndexShardTests.java | 2 ++ .../index/shard/RemoteStoreRefreshListenerTests.java | 11 ++++++++--- .../RemoteStoreReplicationSourceTests.java | 7 ++++++- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da9460f820f96..686ce86463fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020)) - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038)) - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) +- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) - Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) - [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 154e1a4f22242..bc90c42983617 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -3661,6 +3661,10 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro final List internalRefreshListener = new ArrayList<>(); internalRefreshListener.add(new RefreshMetricUpdater(refreshMetric)); + if (this.checkpointPublisher != null && shardRouting.primary() && indexSettings.isSegRepLocalEnabled()) { + internalRefreshListener.add(new CheckpointRefreshListener(this, this.checkpointPublisher)); + } + if (isRemoteStoreEnabled()) { internalRefreshListener.add( new RemoteStoreRefreshListener( @@ -3672,9 +3676,6 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro ); } - if (this.checkpointPublisher != null && shardRouting.primary() && indexSettings.isSegRepLocalEnabled()) { - internalRefreshListener.add(new CheckpointRefreshListener(this, this.checkpointPublisher)); - } /** * With segment replication enabled for primary relocation, recover replica shard initially as read only and * change to a writeable engine during relocation handoff after a round of segment replication. diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index e087bbb265727..2519a4ee48051 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -191,6 +191,8 @@ private synchronized void syncSegments(boolean isRetry) { if (indexShard.getReplicationTracker().isPrimaryMode() == false) { return; } + ReplicationCheckpoint checkpoint = indexShard.getLatestReplicationCheckpoint(); + indexShard.onCheckpointPublished(checkpoint); beforeSegmentsSync(isRetry); long refreshTimeMs = segmentTracker.getLocalRefreshTimeMs(), refreshClockTimeMs = segmentTracker.getLocalRefreshClockTimeMs(); long refreshSeqNo = segmentTracker.getLocalRefreshSeqNo(); @@ -214,7 +216,6 @@ private synchronized void syncSegments(boolean isRetry) { SegmentInfos segmentInfos = segmentInfosGatedCloseable.get(); // Capture replication checkpoint before uploading the segments as upload can take some time and checkpoint can // move. - ReplicationCheckpoint checkpoint = indexShard.getLatestReplicationCheckpoint(); long lastRefreshedCheckpoint = ((InternalEngine) indexShard.getEngine()).lastRefreshedCheckpoint(); Collection localSegmentsPostRefresh = segmentInfos.files(true); diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 8832f6a558b7b..60644ebaf94a3 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2692,6 +2692,7 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc public void testRelocatedForRemoteTranslogBackedIndexWithAsyncDurability() throws IOException { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) @@ -4815,6 +4816,7 @@ public void testTranslogFactoryForReplicaShardWithoutRemoteStore() throws IOExce public void testTranslogFactoryForRemoteTranslogBackedPrimaryShard() throws IOException { Settings primarySettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index ed1438cbc3b08..782450286cb90 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -58,12 +58,17 @@ public class RemoteStoreRefreshListenerTests extends IndexShardTestCase { public void setup(boolean primary, int numberOfDocs) throws IOException { indexShard = newStartedShard( primary, - Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(), + Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(), new InternalEngineFactory() ); - indexDocs(1, numberOfDocs); - indexShard.refresh("test"); + if (primary) { + indexDocs(1, numberOfDocs); + indexShard.refresh("test"); + } clusterService = new ClusterService( Settings.EMPTY, diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java index 04f821a5fc48c..2209514e06438 100644 --- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java @@ -21,6 +21,7 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.common.ReplicationType; import java.io.IOException; import java.util.Collections; @@ -44,9 +45,13 @@ public class RemoteStoreReplicationSourceTests extends OpenSearchIndexLevelRepli @Override public void setUp() throws Exception { super.setUp(); + indexShard = newStartedShard( true, - Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(), + Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(), new InternalEngineFactory() ); From 7dddb3105999d47c3f7dc6e5e16dac2c3821af95 Mon Sep 17 00:00:00 2001 From: Ashish Date: Wed, 12 Jul 2023 06:41:04 +0530 Subject: [PATCH 112/186] Fix remote segments sync retry regression introduced in PR #7119 (#8632) --------- Signed-off-by: Ashish Singh Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../shard/RemoteStoreRefreshListener.java | 18 ++------ .../store/RemoteSegmentStoreDirectory.java | 46 ++++++++++--------- .../RemoteStoreRefreshListenerTests.java | 2 - 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 2519a4ee48051..8f520c0e007bb 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -197,9 +197,9 @@ private synchronized void syncSegments(boolean isRetry) { long refreshTimeMs = segmentTracker.getLocalRefreshTimeMs(), refreshClockTimeMs = segmentTracker.getLocalRefreshClockTimeMs(); long refreshSeqNo = segmentTracker.getLocalRefreshSeqNo(); long bytesBeforeUpload = segmentTracker.getUploadBytesSucceeded(), startTimeInNS = System.nanoTime(); + final AtomicBoolean shouldRetry = new AtomicBoolean(true); try { - if (this.primaryTerm != indexShard.getOperationPrimaryTerm()) { this.primaryTerm = indexShard.getOperationPrimaryTerm(); this.remoteDirectory.init(); @@ -252,7 +252,6 @@ private synchronized void syncSegments(boolean isRetry) { ActionListener segmentUploadsCompletedListener = new LatchedActionListener<>(new ActionListener<>() { @Override public void onResponse(Void unused) { - boolean shouldRetry = true; try { // Start metadata file upload uploadMetadata(localSegmentsPostRefresh, segmentInfos); @@ -266,27 +265,18 @@ public void onResponse(Void unused) { ); // At this point since we have uploaded new segments, segment infos and segment metadata file, // along with marking minSeqNoToKeep, upload has succeeded completely. - shouldRetry = false; + shouldRetry.set(false); } catch (Exception e) { // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried // in the next refresh. This should not affect durability of the indexed data after remote trans-log // integration. logger.warn("Exception in post new segment upload actions", e); - } finally { - doComplete(shouldRetry); } } @Override public void onFailure(Exception e) { logger.warn("Exception while uploading new segments to the remote segment store", e); - doComplete(true); - } - - private void doComplete(boolean shouldRetry) { - // Update the segment tracker with the final upload status as seen at the end - updateFinalUploadStatusInSegmentTracker(shouldRetry == false, bytesBeforeUpload, startTimeInNS); - afterSegmentsSync(isRetry, shouldRetry); } }, latch); @@ -305,6 +295,8 @@ private void doComplete(boolean shouldRetry) { } catch (Throwable t) { logger.error("Exception in RemoteStoreRefreshListener.afterRefresh()", t); } + updateFinalStatusInSegmentTracker(shouldRetry.get() == false, bytesBeforeUpload, startTimeInNS); + afterSegmentsSync(isRetry, shouldRetry.get()); } /** @@ -517,7 +509,7 @@ private void updateLocalSizeMapAndTracker(Collection segmentFiles) { segmentTracker.setLatestLocalFileNameLengthMap(latestFileNameSizeOnLocalMap); } - private void updateFinalUploadStatusInSegmentTracker(boolean uploadStatus, long bytesBeforeUpload, long startTimeInNS) { + private void updateFinalStatusInSegmentTracker(boolean uploadStatus, long bytesBeforeUpload, long startTimeInNS) { if (uploadStatus) { long bytesUploaded = segmentTracker.getUploadBytesSucceeded() - bytesBeforeUpload; long timeTakenInMS = (System.nanoTime() - startTimeInNS) / 1_000_000L; diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 395ecba442e86..d3e8d961337cc 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -581,30 +581,32 @@ public void uploadMetadata( RemoteSegmentMetadata.CURRENT_VERSION ); try { - IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT); - Map uploadedSegments = new HashMap<>(); - for (String file : segmentFiles) { - if (segmentsUploadedToRemoteStore.containsKey(file)) { - uploadedSegments.put(file, segmentsUploadedToRemoteStore.get(file).toString()); - } else { - throw new NoSuchFileException(file); + try (IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT)) { + Map uploadedSegments = new HashMap<>(); + for (String file : segmentFiles) { + if (segmentsUploadedToRemoteStore.containsKey(file)) { + uploadedSegments.put(file, segmentsUploadedToRemoteStore.get(file).toString()); + } else { + throw new NoSuchFileException(file); + } } - } - ByteBuffersDataOutput byteBuffersIndexOutput = new ByteBuffersDataOutput(); - segmentInfosSnapshot.write(new ByteBuffersIndexOutput(byteBuffersIndexOutput, "Snapshot of SegmentInfos", "SegmentInfos")); - byte[] segmentInfoSnapshotByteArray = byteBuffersIndexOutput.toArrayCopy(); - - metadataStreamWrapper.writeStream( - indexOutput, - new RemoteSegmentMetadata( - RemoteSegmentMetadata.fromMapOfStrings(uploadedSegments), - segmentInfoSnapshotByteArray, - primaryTerm, - segmentInfosSnapshot.getGeneration() - ) - ); - indexOutput.close(); + ByteBuffersDataOutput byteBuffersIndexOutput = new ByteBuffersDataOutput(); + segmentInfosSnapshot.write( + new ByteBuffersIndexOutput(byteBuffersIndexOutput, "Snapshot of SegmentInfos", "SegmentInfos") + ); + byte[] segmentInfoSnapshotByteArray = byteBuffersIndexOutput.toArrayCopy(); + + metadataStreamWrapper.writeStream( + indexOutput, + new RemoteSegmentMetadata( + RemoteSegmentMetadata.fromMapOfStrings(uploadedSegments), + segmentInfoSnapshotByteArray, + primaryTerm, + segmentInfosSnapshot.getGeneration() + ) + ); + } storeDirectory.sync(Collections.singleton(metadataFilename)); remoteMetadataDirectory.copyFrom(storeDirectory, metadataFilename, metadataFilename, IOContext.DEFAULT); } finally { diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 782450286cb90..6f38f080e5035 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.tests.store.BaseDirectoryWrapper; -import org.apache.lucene.tests.util.LuceneTestCase; import org.junit.After; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; @@ -48,7 +47,6 @@ import static org.mockito.Mockito.when; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8549") public class RemoteStoreRefreshListenerTests extends IndexShardTestCase { private IndexShard indexShard; private ClusterService clusterService; From c71c7d1be845c99b2471c5674bc2f208f2ca8ef2 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Wed, 12 Jul 2023 19:20:46 +0530 Subject: [PATCH 113/186] Telemetry exporter config (#8624) * Makes exporter configurable Signed-off-by: Gagan Juneja * Makes exporter configurable Signed-off-by: Gagan Juneja * Makes exporter configurable Signed-off-by: Gagan Juneja * Makes exporter configurable Signed-off-by: Gagan Juneja * Makes exporter configurable Signed-off-by: Gagan Juneja * Add logs for successful instantiation Signed-off-by: Gagan Juneja * Move settings to OtelTelemetrySettings Signed-off-by: Gagan Juneja * Move settings to OtelTelemetrySettings Signed-off-by: Gagan Juneja * Move settings to OtelTelemetrySettings Signed-off-by: Gagan Juneja * Add test cases Signed-off-by: Gagan Juneja * Fix test cases Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Fix test cases Signed-off-by: Gagan Juneja * refactor Signed-off-by: Gagan Juneja * refactor Signed-off-by: Gagan Juneja * refactor Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 1 + .../telemetry/OTelTelemetryPlugin.java | 38 ++------- .../telemetry/OTelTelemetrySettings.java | 84 +++++++++++++++++++ .../tracing/OTelResourceProvider.java | 10 +-- .../exporter/OTelSpanExporterFactory.java | 79 +++++++++++++++++ .../tracing/exporter/package-info.java | 12 +++ .../plugin-metadata/plugin-security.policy | 2 + .../telemetry/OTelTelemetryPluginTests.java | 40 ++++++--- .../tracing/exporter/DummySpanExporter.java | 31 +++++++ .../OTelSpanExporterFactoryTests.java | 65 ++++++++++++++ 10 files changed, 315 insertions(+), 47 deletions(-) create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/package-info.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporter.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 686ce86463fbf..ef26caadc48bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919)) - Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909)) - Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) +- Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) - Add wrapper tracer implementation ### Deprecated diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java index 1c38c9dc8d6be..a1ca3adf4d2a2 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java @@ -10,7 +10,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.TelemetryPlugin; import org.opensearch.telemetry.metrics.MetricsTelemetry; @@ -29,36 +28,6 @@ public class OTelTelemetryPlugin extends Plugin implements TelemetryPlugin { static final String OTEL_TRACER_NAME = "otel"; - /** - * span exporter batch size - */ - public static final Setting TRACER_EXPORTER_BATCH_SIZE_SETTING = Setting.intSetting( - "telemetry.otel.tracer.exporter.batch_size", - 512, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - /** - * span exporter max queue size - */ - public static final Setting TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING = Setting.intSetting( - "telemetry.otel.tracer.exporter.max_queue_size", - 2048, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - /** - * span exporter delay in seconds - */ - public static final Setting TRACER_EXPORTER_DELAY_SETTING = Setting.timeSetting( - "telemetry.otel.tracer.exporter.delay", - TimeValue.timeValueSeconds(2), - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - private final Settings settings; /** @@ -71,7 +40,12 @@ public OTelTelemetryPlugin(Settings settings) { @Override public List> getSettings() { - return Arrays.asList(TRACER_EXPORTER_BATCH_SIZE_SETTING, TRACER_EXPORTER_DELAY_SETTING, TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING); + return Arrays.asList( + OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING, + OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING, + OTelTelemetrySettings.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING, + OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING + ); } @Override diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java new file mode 100644 index 0000000000000..2df13e2cd5612 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry; + +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import org.opensearch.SpecialPermission; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.telemetry.tracing.exporter.OTelSpanExporterFactory; + +/** + * OTel specific telemetry settings. + */ +public final class OTelTelemetrySettings { + + /** + * Base Constructor. + */ + private OTelTelemetrySettings() {} + + /** + * span exporter batch size + */ + public static final Setting TRACER_EXPORTER_BATCH_SIZE_SETTING = Setting.intSetting( + "telemetry.otel.tracer.exporter.batch_size", + 512, + 1, + Setting.Property.NodeScope, + Setting.Property.Final + ); + /** + * span exporter max queue size + */ + public static final Setting TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING = Setting.intSetting( + "telemetry.otel.tracer.exporter.max_queue_size", + 2048, + 1, + Setting.Property.NodeScope, + Setting.Property.Final + ); + /** + * span exporter delay in seconds + */ + public static final Setting TRACER_EXPORTER_DELAY_SETTING = Setting.timeSetting( + "telemetry.otel.tracer.exporter.delay", + TimeValue.timeValueSeconds(2), + Setting.Property.NodeScope, + Setting.Property.Final + ); + + /** + * Span Exporter type setting. + */ + @SuppressWarnings("unchecked") + public static final Setting> OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING = new Setting<>( + "telemetry.otel.tracer.span.exporter.class", + LoggingSpanExporter.class.getName(), + className -> { + // Check we ourselves are not being called by unprivileged code. + SpecialPermission.check(); + + try { + return AccessController.doPrivileged((PrivilegedExceptionAction>) () -> { + final ClassLoader loader = OTelSpanExporterFactory.class.getClassLoader(); + return (Class) loader.loadClass(className); + }); + } catch (PrivilegedActionException ex) { + throw new IllegalStateException("Unable to load span exporter class:" + className, ex.getCause()); + } + }, + Setting.Property.NodeScope, + Setting.Property.Final + ); +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java index 292165979c2f2..4d3605ae03993 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java @@ -12,7 +12,6 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.exporter.logging.LoggingSpanExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -23,10 +22,11 @@ import org.opensearch.common.settings.Settings; import java.util.concurrent.TimeUnit; +import org.opensearch.telemetry.tracing.exporter.OTelSpanExporterFactory; -import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_BATCH_SIZE_SETTING; -import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_DELAY_SETTING; -import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING; /** * This class encapsulates all OpenTelemetry related resources @@ -42,7 +42,7 @@ private OTelResourceProvider() {} public static OpenTelemetry get(Settings settings) { return get( settings, - LoggingSpanExporter.create(), + OTelSpanExporterFactory.create(settings), ContextPropagators.create(W3CTraceContextPropagator.getInstance()), Sampler.alwaysOn() ); diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java new file mode 100644 index 0000000000000..c73de4370465f --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.exporter; + +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.SpecialPermission; +import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.OTelTelemetrySettings; + +/** + * Factory class to create the {@link SpanExporter} instance. + */ +public class OTelSpanExporterFactory { + + private static final Logger logger = LogManager.getLogger(OTelSpanExporterFactory.class); + + /** + * Base constructor. + */ + private OTelSpanExporterFactory() { + + } + + /** + * Creates the {@link SpanExporter} instances based on the OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING value. + * As of now, it expects the SpanExporter implementations to have a create factory method to instantiate the + * SpanExporter. + * @param settings settings. + * @return SpanExporter instance. + */ + public static SpanExporter create(Settings settings) { + Class spanExporterProviderClass = OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.get(settings); + SpanExporter spanExporter = instantiateSpanExporter(spanExporterProviderClass); + logger.info("Successfully instantiated the SpanExporter class {}", spanExporterProviderClass); + return spanExporter; + } + + private static SpanExporter instantiateSpanExporter(Class spanExporterProviderClass) { + try { + // Check we ourselves are not being called by unprivileged code. + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + try { + return (SpanExporter) MethodHandles.publicLookup() + .findStatic(spanExporterProviderClass, "create", MethodType.methodType(spanExporterProviderClass)) + .asType(MethodType.methodType(SpanExporter.class)) + .invokeExact(); + } catch (Throwable e) { + if (e.getCause() instanceof NoSuchMethodException) { + throw new IllegalStateException("No create factory method exist in [" + spanExporterProviderClass.getName() + "]"); + } else { + throw new IllegalStateException( + "SpanExporter instantiation failed for class [" + spanExporterProviderClass.getName() + "]", + e.getCause() + ); + } + } + }); + } catch (PrivilegedActionException ex) { + throw new IllegalStateException( + "SpanExporter instantiation failed for class [" + spanExporterProviderClass.getName() + "]", + ex.getCause() + ); + } + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/package-info.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/package-info.java new file mode 100644 index 0000000000000..a5fffadabd75f --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for telemetry. + */ +package org.opensearch.telemetry.tracing.exporter; diff --git a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy index 0f556121915bb..4480cbb2bab4b 100644 --- a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy @@ -7,6 +7,8 @@ */ grant { + permission java.lang.RuntimePermission "getClassLoader"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; }; diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java index c6ffba04ac285..d57ae554a462d 100644 --- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java @@ -8,6 +8,8 @@ package org.opensearch.telemetry; +import org.junit.After; +import org.junit.Before; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -23,28 +25,46 @@ import java.util.Optional; import static org.opensearch.telemetry.OTelTelemetryPlugin.OTEL_TRACER_NAME; -import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_BATCH_SIZE_SETTING; -import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_DELAY_SETTING; -import static org.opensearch.telemetry.OTelTelemetryPlugin.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING; +import static org.opensearch.telemetry.OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING; public class OTelTelemetryPluginTests extends OpenSearchTestCase { + private OTelTelemetryPlugin oTelTracerModulePlugin; + private Optional telemetry; + private TracingTelemetry tracingTelemetry; + + @Before + public void setup() { + // TRACER_EXPORTER_DELAY_SETTING should always be less than 10 seconds because + // io.opentelemetry.sdk.OpenTelemetrySdk.close waits only for 10 seconds for shutdown to complete. + Settings settings = Settings.builder().put(TRACER_EXPORTER_DELAY_SETTING.getKey(), "1s").build(); + oTelTracerModulePlugin = new OTelTelemetryPlugin(settings); + telemetry = oTelTracerModulePlugin.getTelemetry(null); + tracingTelemetry = telemetry.get().getTracingTelemetry(); + } + public void testGetTelemetry() { Set> allTracerSettings = new HashSet<>(); ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add)); - Settings settings = Settings.builder().build(); - OTelTelemetryPlugin oTelTracerModulePlugin = new OTelTelemetryPlugin(settings); - Optional tracer = oTelTracerModulePlugin.getTelemetry(null); - assertEquals(OTEL_TRACER_NAME, oTelTracerModulePlugin.getName()); - TracingTelemetry tracingTelemetry = tracer.get().getTracingTelemetry(); assertTrue(tracingTelemetry instanceof OTelTracingTelemetry); assertEquals( - Arrays.asList(TRACER_EXPORTER_BATCH_SIZE_SETTING, TRACER_EXPORTER_DELAY_SETTING, TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING), + Arrays.asList( + TRACER_EXPORTER_BATCH_SIZE_SETTING, + TRACER_EXPORTER_DELAY_SETTING, + TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING, + OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING + ), oTelTracerModulePlugin.getSettings() ); - tracingTelemetry.close(); } + @After + public void cleanup() { + tracingTelemetry.close(); + } } diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporter.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporter.java new file mode 100644 index 0000000000000..3f250b5aa481f --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporter.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.exporter; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.Collection; + +public class DummySpanExporter implements SpanExporter { + @Override + public CompletableResultCode export(Collection spans) { + return null; + } + + @Override + public CompletableResultCode flush() { + return null; + } + + @Override + public CompletableResultCode shutdown() { + return null; + } +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java new file mode 100644 index 0000000000000..80cba425ed163 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.exporter; + +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.OTelTelemetrySettings; +import org.opensearch.test.OpenSearchTestCase; + +public class OTelSpanExporterFactoryTests extends OpenSearchTestCase { + + public void testSpanExporterDefault() { + Settings settings = Settings.builder().build(); + SpanExporter spanExporter = OTelSpanExporterFactory.create(settings); + assertTrue(spanExporter instanceof LoggingSpanExporter); + } + + public void testSpanExporterLogging() { + Settings settings = Settings.builder() + .put( + OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(), + "io.opentelemetry.exporter.logging.LoggingSpanExporter" + ) + .build(); + SpanExporter spanExporter = OTelSpanExporterFactory.create(settings); + assertTrue(spanExporter instanceof LoggingSpanExporter); + } + + public void testSpanExporterInvalid() { + Settings settings = Settings.builder().put(OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(), "abc").build(); + assertThrows(IllegalArgumentException.class, () -> OTelSpanExporterFactory.create(settings)); + } + + public void testSpanExporterNoCreateFactoryMethod() { + Settings settings = Settings.builder() + .put( + OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(), + "org.opensearch.telemetry.tracing.exporter.DummySpanExporter" + ) + .build(); + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> OTelSpanExporterFactory.create(settings)); + assertEquals( + "SpanExporter instantiation failed for class [org.opensearch.telemetry.tracing.exporter.DummySpanExporter]", + exception.getMessage() + ); + } + + public void testSpanExporterNonSpanExporterClass() { + Settings settings = Settings.builder() + .put(OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(), "java.lang.String") + .build(); + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> OTelSpanExporterFactory.create(settings)); + assertEquals("SpanExporter instantiation failed for class [java.lang.String]", exception.getMessage()); + assertTrue(exception.getCause() instanceof NoSuchMethodError); + + } + +} From c40323b958f3a202f851a5433a19e3f290b5dec0 Mon Sep 17 00:00:00 2001 From: Raghuvansh Raj Date: Wed, 12 Jul 2023 21:59:10 +0530 Subject: [PATCH 114/186] [Remote Store] Add S3 async upload utilities and models (#7217) Signed-off-by: Raghuvansh Raj Co-authored-by: Vikas Bansal --- .../s3/AmazonAsyncS3Reference.java | 41 ++ .../s3/AmazonAsyncS3WithCredentials.java | 52 +++ .../repositories/s3/S3AsyncService.java | 430 ++++++++++++++++++ .../repositories/s3/S3ClientSettings.java | 110 +++++ .../repositories/s3/SocketAccess.java | 2 +- .../s3/async/AsyncExecutorContainer.java | 50 ++ .../s3/async/AsyncPartsHandler.java | 183 ++++++++ .../s3/async/AsyncTransferEventLoopGroup.java | 62 +++ .../s3/async/AsyncTransferManager.java | 354 ++++++++++++++ .../repositories/s3/async/UploadRequest.java | 84 ++++ .../repositories/s3/io/CheckedContainer.java | 31 ++ .../plugin-metadata/plugin-security.policy | 5 +- .../repositories/s3/S3AsyncServiceTests.java | 95 ++++ .../s3/S3BlobContainerRetriesTests.java | 1 + .../s3/async/AsyncTransferManagerTests.java | 238 ++++++++++ .../AbstractBlobContainerRetriesTestCase.java | 83 ---- .../blobstore/ZeroInputStream.java | 96 ++++ 17 files changed, 1831 insertions(+), 86 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncExecutorContainer.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferEventLoopGroup.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/io/CheckedContainer.java create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java create mode 100644 test/framework/src/main/java/org/opensearch/repositories/blobstore/ZeroInputStream.java diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java new file mode 100644 index 0000000000000..258b00bde75f0 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.concurrent.RefCountedReleasable; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +import java.io.Closeable; +import java.io.IOException; + +/** + * Handles the shutdown of the wrapped {@link software.amazon.awssdk.services.s3.S3AsyncClient} using reference + * counting. + */ +public class AmazonAsyncS3Reference extends RefCountedReleasable { + + private static final Logger logger = LogManager.getLogger(AmazonAsyncS3Reference.class); + + AmazonAsyncS3Reference(AmazonAsyncS3WithCredentials client) { + super("AWS_S3_CLIENT", client, () -> { + client.client().close(); + client.priorityClient().close(); + AwsCredentialsProvider credentials = client.credentials(); + if (credentials instanceof Closeable) { + try { + ((Closeable) credentials).close(); + } catch (IOException e) { + logger.error("Exception while closing AwsCredentialsProvider", e); + } + } + }); + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java new file mode 100644 index 0000000000000..15f104f51a067 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.opensearch.common.Nullable; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +/** + * The holder of the AmazonS3 and AWSCredentialsProvider + */ +final class AmazonAsyncS3WithCredentials { + private final S3AsyncClient client; + private final S3AsyncClient priorityClient; + private final AwsCredentialsProvider credentials; + + private AmazonAsyncS3WithCredentials( + final S3AsyncClient client, + final S3AsyncClient priorityClient, + @Nullable final AwsCredentialsProvider credentials + ) { + this.client = client; + this.credentials = credentials; + this.priorityClient = priorityClient; + } + + S3AsyncClient client() { + return client; + } + + S3AsyncClient priorityClient() { + return priorityClient; + } + + AwsCredentialsProvider credentials() { + return credentials; + } + + static AmazonAsyncS3WithCredentials create( + final S3AsyncClient client, + final S3AsyncClient priorityClient, + @Nullable final AwsCredentialsProvider credentials + ) { + return new AmazonAsyncS3WithCredentials(client, priorityClient, credentials); + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java new file mode 100644 index 0000000000000..653034ee9afde --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java @@ -0,0 +1,430 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.Nullable; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; +import org.opensearch.repositories.s3.S3ClientSettings.IrsaCredentials; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.ProxyConfiguration; +import software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup; +import software.amazon.awssdk.profiles.ProfileFileSystemSetting; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.auth.StsWebIdentityTokenFileCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; + +import static java.util.Collections.emptyMap; + +class S3AsyncService implements Closeable { + private static final Logger logger = LogManager.getLogger(S3AsyncService.class); + + private static final String STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY = "aws.stsEndpointOverride"; + + private static final String DEFAULT_S3_ENDPOINT = "s3.amazonaws.com"; + + private volatile Map clientsCache = emptyMap(); + + /** + * Client settings calculated from static configuration and settings in the keystore. + */ + private volatile Map staticClientSettings; + + /** + * Client settings derived from those in {@link #staticClientSettings} by combining them with settings + * in the {@link RepositoryMetadata}. + */ + private volatile Map derivedClientSettings = emptyMap(); + + S3AsyncService(final Path configPath) { + staticClientSettings = MapBuilder.newMapBuilder() + .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default", configPath)) + .immutableMap(); + } + + /** + * Refreshes the settings for the AmazonS3 clients and clears the cache of + * existing clients. New clients will be build using these new settings. Old + * clients are usable until released. On release they will be destroyed instead + * of being returned to the cache. + */ + public synchronized void refreshAndClearCache(Map clientsSettings) { + // shutdown all unused clients + // others will shutdown on their respective release + releaseCachedClients(); + this.staticClientSettings = MapBuilder.newMapBuilder(clientsSettings).immutableMap(); + derivedClientSettings = emptyMap(); + assert this.staticClientSettings.containsKey("default") : "always at least have 'default'"; + // clients are built lazily by {@link client} + } + + /** + * Attempts to retrieve a client by its repository metadata and settings from the cache. + * If the client does not exist it will be created. + */ + public AmazonAsyncS3Reference client( + RepositoryMetadata repositoryMetadata, + AsyncExecutorContainer priorityExecutorBuilder, + AsyncExecutorContainer normalExecutorBuilder + ) { + final S3ClientSettings clientSettings = settings(repositoryMetadata); + { + final AmazonAsyncS3Reference clientReference = clientsCache.get(clientSettings); + if (clientReference != null && clientReference.tryIncRef()) { + return clientReference; + } + } + synchronized (this) { + final AmazonAsyncS3Reference existing = clientsCache.get(clientSettings); + if (existing != null && existing.tryIncRef()) { + return existing; + } + final AmazonAsyncS3Reference clientReference = new AmazonAsyncS3Reference( + buildClient(clientSettings, priorityExecutorBuilder, normalExecutorBuilder) + ); + clientReference.incRef(); + clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientSettings, clientReference).immutableMap(); + return clientReference; + } + } + + /** + * Either fetches {@link S3ClientSettings} for a given {@link RepositoryMetadata} from cached settings or creates them + * by overriding static client settings from {@link #staticClientSettings} with settings found in the repository metadata. + * @param repositoryMetadata Repository Metadata + * @return S3ClientSettings + */ + S3ClientSettings settings(RepositoryMetadata repositoryMetadata) { + final Settings settings = repositoryMetadata.settings(); + { + final S3ClientSettings existing = derivedClientSettings.get(settings); + if (existing != null) { + return existing; + } + } + final String clientName = S3Repository.CLIENT_NAME.get(settings); + final S3ClientSettings staticSettings = staticClientSettings.get(clientName); + if (staticSettings != null) { + synchronized (this) { + final S3ClientSettings existing = derivedClientSettings.get(settings); + if (existing != null) { + return existing; + } + final S3ClientSettings newSettings = staticSettings.refine(settings); + derivedClientSettings = MapBuilder.newMapBuilder(derivedClientSettings).put(settings, newSettings).immutableMap(); + return newSettings; + } + } + throw new IllegalArgumentException( + "Unknown s3 client name [" + + clientName + + "]. Existing client configs: " + + Strings.collectionToDelimitedString(staticClientSettings.keySet(), ",") + ); + } + + // proxy for testing + synchronized AmazonAsyncS3WithCredentials buildClient( + final S3ClientSettings clientSettings, + AsyncExecutorContainer priorityExecutorBuilder, + AsyncExecutorContainer normalExecutorBuilder + ) { + setDefaultAwsProfilePath(); + final S3AsyncClientBuilder builder = S3AsyncClient.builder(); + builder.overrideConfiguration(buildOverrideConfiguration(clientSettings)); + final AwsCredentialsProvider credentials = buildCredentials(logger, clientSettings); + builder.credentialsProvider(credentials); + + String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : DEFAULT_S3_ENDPOINT; + if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) { + // Manually add the schema to the endpoint to work around https://github.com/aws/aws-sdk-java/issues/2274 + endpoint = clientSettings.protocol.toString() + "://" + endpoint; + } + logger.debug("using endpoint [{}] and region [{}]", endpoint, clientSettings.region); + + // If the endpoint configuration isn't set on the builder then the default behaviour is to try + // and work out what region we are in and use an appropriate endpoint - see AwsClientBuilder#setRegion. + // In contrast, directly-constructed clients use s3.amazonaws.com unless otherwise instructed. We currently + // use a directly-constructed client, and need to keep the existing behaviour to avoid a breaking change, + // so to move to using the builder we must set it explicitly to keep the existing behaviour. + // + // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) + // so this change removes that usage of a deprecated API. + builder.endpointOverride(URI.create(endpoint)); + builder.region(Region.of(clientSettings.region)); + if (clientSettings.pathStyleAccess) { + builder.forcePathStyle(true); + } + + builder.httpClient(buildHttpClient(clientSettings, priorityExecutorBuilder.getAsyncTransferEventLoopGroup())); + builder.asyncConfiguration( + ClientAsyncConfiguration.builder() + .advancedOption( + SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + priorityExecutorBuilder.getFutureCompletionExecutor() + ) + .build() + ); + final S3AsyncClient priorityClient = SocketAccess.doPrivileged(builder::build); + + builder.httpClient(buildHttpClient(clientSettings, normalExecutorBuilder.getAsyncTransferEventLoopGroup())); + builder.asyncConfiguration( + ClientAsyncConfiguration.builder() + .advancedOption( + SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + normalExecutorBuilder.getFutureCompletionExecutor() + ) + .build() + ); + final S3AsyncClient client = SocketAccess.doPrivileged(builder::build); + + return AmazonAsyncS3WithCredentials.create(client, priorityClient, credentials); + } + + static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSettings clientSettings) { + return ClientOverrideConfiguration.builder() + .retryPolicy( + RetryPolicy.builder() + .numRetries(clientSettings.maxRetries) + .throttlingBackoffStrategy( + clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy() : BackoffStrategy.none() + ) + .build() + ) + .apiCallAttemptTimeout(Duration.ofMillis(clientSettings.requestTimeoutMillis)) + .build(); + } + + // pkg private for tests + static SdkAsyncHttpClient buildHttpClient(S3ClientSettings clientSettings, AsyncTransferEventLoopGroup asyncTransferEventLoopGroup) { + // the response metadata cache is only there for diagnostics purposes, + // but can force objects from every response to the old generation. + NettyNioAsyncHttpClient.Builder clientBuilder = NettyNioAsyncHttpClient.builder(); + + if (clientSettings.proxySettings.getType() != ProxySettings.ProxyType.DIRECT) { + ProxyConfiguration.Builder proxyConfiguration = ProxyConfiguration.builder(); + proxyConfiguration.scheme(clientSettings.proxySettings.getType().toProtocol().toString()); + proxyConfiguration.host(clientSettings.proxySettings.getHostName()); + proxyConfiguration.port(clientSettings.proxySettings.getPort()); + proxyConfiguration.username(clientSettings.proxySettings.getUsername()); + proxyConfiguration.password(clientSettings.proxySettings.getPassword()); + clientBuilder.proxyConfiguration(proxyConfiguration.build()); + } + + // TODO: add max retry and UseThrottleRetry. Replace values with settings and put these in default settings + clientBuilder.connectionTimeout(Duration.ofMillis(clientSettings.connectionTimeoutMillis)); + clientBuilder.connectionAcquisitionTimeout(Duration.ofMillis(clientSettings.connectionAcquisitionTimeoutMillis)); + clientBuilder.maxPendingConnectionAcquires(10_000); + clientBuilder.maxConcurrency(clientSettings.maxConnections); + clientBuilder.eventLoopGroup(SdkEventLoopGroup.create(asyncTransferEventLoopGroup.getEventLoopGroup())); + clientBuilder.tcpKeepAlive(true); + + return clientBuilder.build(); + } + + // pkg private for tests + static AwsCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { + final AwsCredentials basicCredentials = clientSettings.credentials; + final IrsaCredentials irsaCredentials = buildFromEnvironment(clientSettings.irsaCredentials); + + // If IAM Roles for Service Accounts (IRSA) credentials are configured, start with them first + if (irsaCredentials != null) { + logger.debug("Using IRSA credentials"); + + final Region region = Region.of(clientSettings.region); + StsClient stsClient = SocketAccess.doPrivileged(() -> { + StsClientBuilder builder = StsClient.builder().region(region); + + final String stsEndpoint = System.getProperty(STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY); + if (stsEndpoint != null) { + builder = builder.endpointOverride(URI.create(stsEndpoint)); + } + + if (basicCredentials != null) { + builder = builder.credentialsProvider(StaticCredentialsProvider.create(basicCredentials)); + } else { + builder = builder.credentialsProvider(DefaultCredentialsProvider.create()); + } + + return builder.build(); + }); + + if (irsaCredentials.getIdentityTokenFile() == null) { + final StsAssumeRoleCredentialsProvider.Builder stsCredentialsProviderBuilder = StsAssumeRoleCredentialsProvider.builder() + .stsClient(stsClient) + .refreshRequest( + AssumeRoleRequest.builder() + .roleArn(irsaCredentials.getRoleArn()) + .roleSessionName(irsaCredentials.getRoleSessionName()) + .build() + ); + + final StsAssumeRoleCredentialsProvider stsCredentialsProvider = SocketAccess.doPrivileged( + stsCredentialsProviderBuilder::build + ); + + return new PrivilegedSTSAssumeRoleSessionCredentialsProvider<>(stsClient, stsCredentialsProvider); + } else { + final StsWebIdentityTokenFileCredentialsProvider.Builder stsCredentialsProviderBuilder = + StsWebIdentityTokenFileCredentialsProvider.builder() + .stsClient(stsClient) + .roleArn(irsaCredentials.getRoleArn()) + .roleSessionName(irsaCredentials.getRoleSessionName()) + .webIdentityTokenFile(Path.of(irsaCredentials.getIdentityTokenFile())); + + final StsWebIdentityTokenFileCredentialsProvider stsCredentialsProvider = SocketAccess.doPrivileged( + stsCredentialsProviderBuilder::build + ); + + return new PrivilegedSTSAssumeRoleSessionCredentialsProvider<>(stsClient, stsCredentialsProvider); + } + } else if (basicCredentials != null) { + logger.debug("Using basic key/secret credentials"); + return StaticCredentialsProvider.create(basicCredentials); + } else { + logger.debug("Using instance profile credentials"); + return new PrivilegedInstanceProfileCredentialsProvider(); + } + } + + // Aws v2 sdk tries to load a default profile from home path which is restricted. Hence, setting these to random + // valid paths. + @SuppressForbidden(reason = "Need to provide this override to v2 SDK so that path does not default to home path") + private static void setDefaultAwsProfilePath() { + if (ProfileFileSystemSetting.AWS_SHARED_CREDENTIALS_FILE.getStringValue().isEmpty()) { + System.setProperty(ProfileFileSystemSetting.AWS_SHARED_CREDENTIALS_FILE.property(), System.getProperty("opensearch.path.conf")); + } + if (ProfileFileSystemSetting.AWS_CONFIG_FILE.getStringValue().isEmpty()) { + System.setProperty(ProfileFileSystemSetting.AWS_CONFIG_FILE.property(), System.getProperty("opensearch.path.conf")); + } + } + + private static IrsaCredentials buildFromEnvironment(IrsaCredentials defaults) { + if (defaults == null) { + return null; + } + + String webIdentityTokenFile = defaults.getIdentityTokenFile(); + if (webIdentityTokenFile == null) { + webIdentityTokenFile = System.getenv(SdkSystemSetting.AWS_WEB_IDENTITY_TOKEN_FILE.environmentVariable()); + } + + String roleArn = defaults.getRoleArn(); + if (roleArn == null) { + roleArn = System.getenv(SdkSystemSetting.AWS_ROLE_ARN.environmentVariable()); + } + + String roleSessionName = defaults.getRoleSessionName(); + if (roleSessionName == null) { + roleSessionName = System.getenv(SdkSystemSetting.AWS_ROLE_SESSION_NAME.environmentVariable()); + } + + return new IrsaCredentials(webIdentityTokenFile, roleArn, roleSessionName); + } + + private synchronized void releaseCachedClients() { + // the clients will shutdown when they will not be used anymore + for (final AmazonAsyncS3Reference clientReference : clientsCache.values()) { + clientReference.decRef(); + } + + // clear previously cached clients, they will be build lazily + clientsCache = emptyMap(); + derivedClientSettings = emptyMap(); + } + + static class PrivilegedInstanceProfileCredentialsProvider implements AwsCredentialsProvider { + private final AwsCredentialsProvider credentials; + + private PrivilegedInstanceProfileCredentialsProvider() { + this.credentials = initializeProvider(); + } + + private AwsCredentialsProvider initializeProvider() { + if (SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI.getStringValue().isPresent() + || SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_FULL_URI.getStringValue().isPresent()) { + + return ContainerCredentialsProvider.builder().asyncCredentialUpdateEnabled(true).build(); + } + // InstanceProfileCredentialsProvider as last item of chain + return InstanceProfileCredentialsProvider.builder().asyncCredentialUpdateEnabled(true).build(); + } + + @Override + public AwsCredentials resolveCredentials() { + return SocketAccess.doPrivileged(credentials::resolveCredentials); + } + } + + static class PrivilegedSTSAssumeRoleSessionCredentialsProvider

+ implements + AwsCredentialsProvider, + Closeable { + private final P credentials; + private final StsClient stsClient; + + private PrivilegedSTSAssumeRoleSessionCredentialsProvider(@Nullable final StsClient stsClient, final P credentials) { + this.stsClient = stsClient; + this.credentials = credentials; + } + + @Override + public void close() throws IOException { + SocketAccess.doPrivilegedIOException(() -> { + credentials.close(); + if (stsClient != null) { + stsClient.close(); + } + return null; + }); + } + + @Override + public AwsCredentials resolveCredentials() { + return SocketAccess.doPrivileged(credentials::resolveCredentials); + } + } + + @Override + public void close() { + releaseCachedClients(); + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 5f6be6ac01e76..25e9797018102 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -172,6 +172,48 @@ final class S3ClientSettings { key -> Setting.timeSetting(key, TimeValue.timeValueMillis(50_000), Property.NodeScope) ); + /** The request timeout for connecting to s3. */ + static final Setting.AffixSetting REQUEST_TIMEOUT_SETTING = Setting.affixKeySetting( + PREFIX, + "request_timeout", + key -> Setting.timeSetting(key, TimeValue.timeValueMinutes(2), Property.NodeScope) + ); + + /** The connection timeout for connecting to s3. */ + static final Setting.AffixSetting CONNECTION_TIMEOUT_SETTING = Setting.affixKeySetting( + PREFIX, + "connection_timeout", + key -> Setting.timeSetting(key, TimeValue.timeValueSeconds(10), Property.NodeScope) + ); + + /** The connection TTL for connecting to s3. */ + static final Setting.AffixSetting CONNECTION_TTL_SETTING = Setting.affixKeySetting( + PREFIX, + "connection_ttl", + key -> Setting.timeSetting(key, TimeValue.timeValueMillis(5000), Property.NodeScope) + ); + + /** The maximum connections to s3. */ + static final Setting.AffixSetting MAX_CONNECTIONS_SETTING = Setting.affixKeySetting( + PREFIX, + "max_connections", + key -> Setting.intSetting(key, 100, Property.NodeScope) + ); + + /** Connection acquisition timeout for new connections to S3. */ + static final Setting.AffixSetting CONNECTION_ACQUISITION_TIMEOUT = Setting.affixKeySetting( + PREFIX, + "connection_acquisition_timeout", + key -> Setting.timeSetting(key, TimeValue.timeValueMinutes(2), Property.NodeScope) + ); + + /** The maximum pending connections to S3. */ + static final Setting.AffixSetting MAX_PENDING_CONNECTION_ACQUIRES = Setting.affixKeySetting( + PREFIX, + "max_pending_connection_acquires", + key -> Setting.intSetting(key, 10_000, Property.NodeScope) + ); + /** The number of retries to use when an s3 request fails. */ static final Setting.AffixSetting MAX_RETRIES_SETTING = Setting.affixKeySetting( PREFIX, @@ -232,6 +274,21 @@ final class S3ClientSettings { /** The read timeout for the s3 client. */ final int readTimeoutMillis; + /** The request timeout for the s3 client */ + final int requestTimeoutMillis; + + /** The connection timeout for the s3 client */ + final int connectionTimeoutMillis; + + /** The connection TTL for the s3 client */ + final int connectionTTLMillis; + + /** The max number of connections for the s3 client */ + final int maxConnections; + + /** The connnection acquisition timeout for the s3 async client */ + final int connectionAcquisitionTimeoutMillis; + /** The number of retries to use for the s3 client. */ final int maxRetries; @@ -256,6 +313,11 @@ private S3ClientSettings( String endpoint, Protocol protocol, int readTimeoutMillis, + int requestTimeoutMillis, + int connectionTimeoutMillis, + int connectionTTLMillis, + int maxConnections, + int connectionAcquisitionTimeoutMillis, int maxRetries, boolean throttleRetries, boolean pathStyleAccess, @@ -269,6 +331,11 @@ private S3ClientSettings( this.endpoint = endpoint; this.protocol = protocol; this.readTimeoutMillis = readTimeoutMillis; + this.requestTimeoutMillis = requestTimeoutMillis; + this.connectionTimeoutMillis = connectionTimeoutMillis; + this.connectionTTLMillis = connectionTTLMillis; + this.maxConnections = maxConnections; + this.connectionAcquisitionTimeoutMillis = connectionAcquisitionTimeoutMillis; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; this.pathStyleAccess = pathStyleAccess; @@ -300,6 +367,24 @@ S3ClientSettings refine(Settings repositorySettings) { final int newReadTimeoutMillis = Math.toIntExact( getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis() ); + final int newRequestTimeoutMillis = Math.toIntExact( + getRepoSettingOrDefault(REQUEST_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(requestTimeoutMillis)).millis() + ); + final int newConnectionTimeoutMillis = Math.toIntExact( + getRepoSettingOrDefault(CONNECTION_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(connectionTimeoutMillis)) + .millis() + ); + final int newConnectionTTLMillis = Math.toIntExact( + getRepoSettingOrDefault(CONNECTION_TTL_SETTING, normalizedSettings, TimeValue.timeValueMillis(connectionTTLMillis)).millis() + ); + final int newConnectionAcquisitionTimeoutMillis = Math.toIntExact( + getRepoSettingOrDefault( + CONNECTION_ACQUISITION_TIMEOUT, + normalizedSettings, + TimeValue.timeValueMillis(connectionAcquisitionTimeoutMillis) + ).millis() + ); + final int newMaxConnections = Math.toIntExact(getRepoSettingOrDefault(MAX_CONNECTIONS_SETTING, normalizedSettings, maxConnections)); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); final boolean newPathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); @@ -321,6 +406,11 @@ S3ClientSettings refine(Settings repositorySettings) { && Objects.equals(proxySettings.getHostName(), newProxyHost) && proxySettings.getPort() == newProxyPort && newReadTimeoutMillis == readTimeoutMillis + && newRequestTimeoutMillis == requestTimeoutMillis + && newConnectionTimeoutMillis == connectionTimeoutMillis + && newConnectionTTLMillis == connectionTTLMillis + && newMaxConnections == maxConnections + && newConnectionAcquisitionTimeoutMillis == connectionAcquisitionTimeoutMillis && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials) @@ -338,6 +428,11 @@ S3ClientSettings refine(Settings repositorySettings) { newEndpoint, newProtocol, newReadTimeoutMillis, + newRequestTimeoutMillis, + newConnectionTimeoutMillis, + newConnectionTTLMillis, + newMaxConnections, + newConnectionAcquisitionTimeoutMillis, newMaxRetries, newThrottleRetries, newPathStyleAccess, @@ -463,6 +558,11 @@ static S3ClientSettings getClientSettings(final Settings settings, final String getConfigValue(settings, clientName, ENDPOINT_SETTING), awsProtocol, Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), + Math.toIntExact(getConfigValue(settings, clientName, REQUEST_TIMEOUT_SETTING).millis()), + Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_TIMEOUT_SETTING).millis()), + Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_TTL_SETTING).millis()), + Math.toIntExact(getConfigValue(settings, clientName, MAX_CONNECTIONS_SETTING)), + Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_ACQUISITION_TIMEOUT).millis()), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS), @@ -532,6 +632,11 @@ public boolean equals(final Object o) { } final S3ClientSettings that = (S3ClientSettings) o; return readTimeoutMillis == that.readTimeoutMillis + && requestTimeoutMillis == that.requestTimeoutMillis + && connectionTimeoutMillis == that.connectionTimeoutMillis + && connectionTTLMillis == that.connectionTTLMillis + && maxConnections == that.maxConnections + && connectionAcquisitionTimeoutMillis == that.connectionAcquisitionTimeoutMillis && maxRetries == that.maxRetries && throttleRetries == that.throttleRetries && Objects.equals(credentials, that.credentials) @@ -552,6 +657,11 @@ public int hashCode() { protocol, proxySettings, readTimeoutMillis, + requestTimeoutMillis, + connectionTimeoutMillis, + connectionTTLMillis, + maxConnections, + connectionAcquisitionTimeoutMillis, maxRetries, throttleRetries, disableChunkedEncoding, diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/SocketAccess.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/SocketAccess.java index 0a6408764aeeb..4888764dbc720 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/SocketAccess.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/SocketAccess.java @@ -46,7 +46,7 @@ * {@link SocketPermission} 'connect' to establish connections. This class wraps the operations requiring access in * {@link AccessController#doPrivileged(PrivilegedAction)} blocks. */ -final class SocketAccess { +public final class SocketAccess { private SocketAccess() {} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncExecutorContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncExecutorContainer.java new file mode 100644 index 0000000000000..1ae1a15ad4010 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncExecutorContainer.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import java.util.concurrent.ExecutorService; + +/** + * An encapsulation for the {@link AsyncTransferEventLoopGroup}, and the stream reader and future completion executor services + */ +public class AsyncExecutorContainer { + + private final ExecutorService futureCompletionExecutor; + private final ExecutorService streamReader; + private final AsyncTransferEventLoopGroup asyncTransferEventLoopGroup; + + /** + * Construct a new AsyncExecutorBuilder object + * + * @param futureCompletionExecutor An {@link ExecutorService} to pass to {@link software.amazon.awssdk.services.s3.S3AsyncClient} for future completion + * @param streamReader An {@link ExecutorService} to read streams for upload + * @param asyncTransferEventLoopGroup A {@link AsyncTransferEventLoopGroup} which encapsulates the netty {@link io.netty.channel.EventLoopGroup} for async uploads + */ + public AsyncExecutorContainer( + ExecutorService futureCompletionExecutor, + ExecutorService streamReader, + AsyncTransferEventLoopGroup asyncTransferEventLoopGroup + ) { + this.asyncTransferEventLoopGroup = asyncTransferEventLoopGroup; + this.streamReader = streamReader; + this.futureCompletionExecutor = futureCompletionExecutor; + } + + public ExecutorService getFutureCompletionExecutor() { + return futureCompletionExecutor; + } + + public AsyncTransferEventLoopGroup getAsyncTransferEventLoopGroup() { + return asyncTransferEventLoopGroup; + } + + public ExecutorService getStreamReader() { + return streamReader; + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java new file mode 100644 index 0000000000000..b6af91a08ac2b --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java @@ -0,0 +1,183 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.repositories.s3.SocketAccess; +import org.opensearch.repositories.s3.io.CheckedContainer; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; +import software.amazon.awssdk.utils.CompletableFutureUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReferenceArray; + +/** + * Responsible for handling parts of the original multipart request + */ +public class AsyncPartsHandler { + + private static Logger log = LogManager.getLogger(AsyncPartsHandler.class); + + /** + * Uploads parts of the upload multipart request* + * @param s3AsyncClient S3 client to use for upload + * @param executorService Thread pool for regular upload + * @param priorityExecutorService Thread pool for priority uploads + * @param uploadRequest request for upload + * @param streamContext Stream context used in supplying individual file parts + * @param uploadId Upload Id against which multi-part is being performed + * @param completedParts Reference of completed parts + * @param inputStreamContainers Checksum containers + * @return list of completable futures + * @throws IOException thrown in case of an IO error + */ + public static List> uploadParts( + S3AsyncClient s3AsyncClient, + ExecutorService executorService, + ExecutorService priorityExecutorService, + UploadRequest uploadRequest, + StreamContext streamContext, + String uploadId, + AtomicReferenceArray completedParts, + AtomicReferenceArray inputStreamContainers + ) throws IOException { + List> futures = new ArrayList<>(); + for (int partIdx = 0; partIdx < streamContext.getNumberOfParts(); partIdx++) { + InputStreamContainer inputStreamContainer = streamContext.provideStream(partIdx); + inputStreamContainers.set(partIdx, new CheckedContainer(inputStreamContainer.getContentLength())); + UploadPartRequest.Builder uploadPartRequestBuilder = UploadPartRequest.builder() + .bucket(uploadRequest.getBucket()) + .partNumber(partIdx + 1) + .key(uploadRequest.getKey()) + .uploadId(uploadId) + .contentLength(inputStreamContainer.getContentLength()); + if (uploadRequest.doRemoteDataIntegrityCheck()) { + uploadPartRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32); + } + uploadPart( + s3AsyncClient, + executorService, + priorityExecutorService, + completedParts, + inputStreamContainers, + futures, + uploadPartRequestBuilder.build(), + inputStreamContainer, + uploadRequest + ); + } + + return futures; + } + + /** + * Cleans up parts of the original multipart request* + * @param s3AsyncClient s3 client to use + * @param uploadRequest upload request + * @param uploadId upload id against which multi-part was carried out. + */ + public static void cleanUpParts(S3AsyncClient s3AsyncClient, UploadRequest uploadRequest, String uploadId) { + + AbortMultipartUploadRequest abortMultipartUploadRequest = AbortMultipartUploadRequest.builder() + .bucket(uploadRequest.getBucket()) + .key(uploadRequest.getKey()) + .uploadId(uploadId) + .build(); + SocketAccess.doPrivileged(() -> s3AsyncClient.abortMultipartUpload(abortMultipartUploadRequest).exceptionally(throwable -> { + log.warn( + () -> new ParameterizedMessage( + "Failed to abort previous multipart upload " + + "(id: {})" + + ". You may need to call " + + "S3AsyncClient#abortMultiPartUpload to " + + "free all storage consumed by" + + " all parts. ", + uploadId + ), + throwable + ); + return null; + })); + } + + private static void uploadPart( + S3AsyncClient s3AsyncClient, + ExecutorService executorService, + ExecutorService priorityExecutorService, + AtomicReferenceArray completedParts, + AtomicReferenceArray inputStreamContainers, + List> futures, + UploadPartRequest uploadPartRequest, + InputStreamContainer inputStreamContainer, + UploadRequest uploadRequest + ) { + Integer partNumber = uploadPartRequest.partNumber(); + + ExecutorService streamReadExecutor = uploadRequest.getWritePriority() == WritePriority.HIGH + ? priorityExecutorService + : executorService; + CompletableFuture uploadPartResponseFuture = SocketAccess.doPrivileged( + () -> s3AsyncClient.uploadPart( + uploadPartRequest, + AsyncRequestBody.fromInputStream( + inputStreamContainer.getInputStream(), + inputStreamContainer.getContentLength(), + streamReadExecutor + ) + ) + ); + + CompletableFuture convertFuture = uploadPartResponseFuture.thenApply( + uploadPartResponse -> convertUploadPartResponse( + completedParts, + inputStreamContainers, + uploadPartResponse, + partNumber, + uploadRequest.doRemoteDataIntegrityCheck() + ) + ); + futures.add(convertFuture); + + CompletableFutureUtils.forwardExceptionTo(convertFuture, uploadPartResponseFuture); + } + + private static CompletedPart convertUploadPartResponse( + AtomicReferenceArray completedParts, + AtomicReferenceArray inputStreamContainers, + UploadPartResponse partResponse, + int partNumber, + boolean isRemoteDataIntegrityCheckEnabled + ) { + CompletedPart.Builder completedPartBuilder = CompletedPart.builder().eTag(partResponse.eTag()).partNumber(partNumber); + if (isRemoteDataIntegrityCheckEnabled) { + completedPartBuilder.checksumCRC32(partResponse.checksumCRC32()); + CheckedContainer inputStreamCRC32Container = inputStreamContainers.get(partNumber - 1); + inputStreamCRC32Container.setChecksum(partResponse.checksumCRC32()); + inputStreamContainers.set(partNumber - 1, inputStreamCRC32Container); + } + CompletedPart completedPart = completedPartBuilder.build(); + completedParts.set(partNumber - 1, completedPart); + return completedPart; + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferEventLoopGroup.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferEventLoopGroup.java new file mode 100644 index 0000000000000..381a9671d669a --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferEventLoopGroup.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.Epoll; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.util.concurrent.Future; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.repositories.s3.SocketAccess; + +import java.io.Closeable; +import java.util.concurrent.TimeUnit; + +/** + * AsyncTransferEventLoopGroup is an encapsulation for netty {@link EventLoopGroup} + */ +public class AsyncTransferEventLoopGroup implements Closeable { + private static final String THREAD_PREFIX = "s3-async-transfer-worker"; + private final Logger logger = LogManager.getLogger(AsyncTransferEventLoopGroup.class); + + private final EventLoopGroup eventLoopGroup; + + /** + * Construct a new AsyncTransferEventLoopGroup + * + * @param eventLoopThreads The number of event loop threads for this event loop group + */ + public AsyncTransferEventLoopGroup(int eventLoopThreads) { + // Epoll event loop incurs less GC and provides better performance than Nio loop. Therefore, + // using epoll wherever available is preferred. + this.eventLoopGroup = SocketAccess.doPrivileged( + () -> Epoll.isAvailable() + ? new EpollEventLoopGroup(eventLoopThreads, OpenSearchExecutors.daemonThreadFactory(THREAD_PREFIX)) + : new NioEventLoopGroup(eventLoopThreads, OpenSearchExecutors.daemonThreadFactory(THREAD_PREFIX)) + ); + } + + public EventLoopGroup getEventLoopGroup() { + return eventLoopGroup; + } + + @Override + public void close() { + Future shutdownFuture = eventLoopGroup.shutdownGracefully(0, 5, TimeUnit.SECONDS); + shutdownFuture.awaitUninterruptibly(); + if (!shutdownFuture.isSuccess()) { + logger.warn(new ParameterizedMessage("Error closing {} netty event loop group", THREAD_PREFIX), shutdownFuture.cause()); + } + } + +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java new file mode 100644 index 0000000000000..5b43ae84c51dc --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java @@ -0,0 +1,354 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import com.jcraft.jzlib.JZlib; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.ExceptionsHelper; +import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.exception.CorruptFileException; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.common.util.ByteUtils; +import org.opensearch.repositories.s3.io.CheckedContainer; +import org.opensearch.repositories.s3.SocketAccess; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.http.HttpStatusCode; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.utils.CompletableFutureUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +/** + * A helper class that automatically uses multipart upload based on the size of the source object + */ +public final class AsyncTransferManager { + private static final Logger log = LogManager.getLogger(AsyncTransferManager.class); + private final ExecutorService executorService; + private final ExecutorService priorityExecutorService; + private final long minimumPartSize; + + /** + * The max number of parts on S3 side is 10,000 + */ + private static final long MAX_UPLOAD_PARTS = 10_000; + + /** + * Construct a new object of AsyncTransferManager + * + * @param minimumPartSize The minimum part size for parallel multipart uploads + * @param executorService The stream reader {@link ExecutorService} for normal priority uploads + * @param priorityExecutorService The stream read {@link ExecutorService} for high priority uploads + */ + public AsyncTransferManager(long minimumPartSize, ExecutorService executorService, ExecutorService priorityExecutorService) { + this.executorService = executorService; + this.priorityExecutorService = priorityExecutorService; + this.minimumPartSize = minimumPartSize; + } + + /** + * Upload an object to S3 using the async client + * + * @param s3AsyncClient S3 client to use for upload + * @param uploadRequest The {@link UploadRequest} object encapsulating all relevant details for upload + * @param streamContext The {@link StreamContext} to supply streams during upload + * @return A {@link CompletableFuture} to listen for upload completion + */ + public CompletableFuture uploadObject(S3AsyncClient s3AsyncClient, UploadRequest uploadRequest, StreamContext streamContext) { + + CompletableFuture returnFuture = new CompletableFuture<>(); + try { + if (streamContext.getNumberOfParts() == 1) { + log.debug(() -> "Starting the upload as a single upload part request"); + uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext.provideStream(0), returnFuture); + } else { + log.debug(() -> "Starting the upload as multipart upload request"); + uploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture); + } + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return returnFuture; + } + + private void uploadInParts( + S3AsyncClient s3AsyncClient, + UploadRequest uploadRequest, + StreamContext streamContext, + CompletableFuture returnFuture + ) { + + CreateMultipartUploadRequest.Builder createMultipartUploadRequestBuilder = CreateMultipartUploadRequest.builder() + .bucket(uploadRequest.getBucket()) + .key(uploadRequest.getKey()); + if (uploadRequest.doRemoteDataIntegrityCheck()) { + createMultipartUploadRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32); + } + CompletableFuture createMultipartUploadFuture = SocketAccess.doPrivileged( + () -> s3AsyncClient.createMultipartUpload(createMultipartUploadRequestBuilder.build()) + ); + + // Ensure cancellations are forwarded to the createMultipartUploadFuture future + CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture); + + createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> { + if (throwable != null) { + handleException(returnFuture, () -> "Failed to initiate multipart upload", throwable); + } else { + log.debug(() -> "Initiated new multipart upload, uploadId: " + createMultipartUploadResponse.uploadId()); + doUploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture, createMultipartUploadResponse.uploadId()); + } + }); + } + + private void doUploadInParts( + S3AsyncClient s3AsyncClient, + UploadRequest uploadRequest, + StreamContext streamContext, + CompletableFuture returnFuture, + String uploadId + ) { + + // The list of completed parts must be sorted + AtomicReferenceArray completedParts = new AtomicReferenceArray<>(streamContext.getNumberOfParts()); + AtomicReferenceArray inputStreamContainers = new AtomicReferenceArray<>(streamContext.getNumberOfParts()); + + List> futures; + try { + futures = AsyncPartsHandler.uploadParts( + s3AsyncClient, + executorService, + priorityExecutorService, + uploadRequest, + streamContext, + uploadId, + completedParts, + inputStreamContainers + ); + } catch (Exception ex) { + try { + AsyncPartsHandler.cleanUpParts(s3AsyncClient, uploadRequest, uploadId); + } finally { + returnFuture.completeExceptionally(ex); + } + return; + } + + CompletableFutureUtils.allOfExceptionForwarded(futures.toArray(CompletableFuture[]::new)).thenApply(resp -> { + try { + uploadRequest.getUploadFinalizer().accept(true); + } catch (IOException e) { + throw new RuntimeException(e); + } + return resp; + }).thenApply(ignore -> { + if (uploadRequest.doRemoteDataIntegrityCheck()) { + mergeAndVerifyChecksum(inputStreamContainers, uploadRequest.getKey(), uploadRequest.getExpectedChecksum()); + } + return null; + }) + .thenCompose(ignore -> completeMultipartUpload(s3AsyncClient, uploadRequest, uploadId, completedParts)) + .handle(handleExceptionOrResponse(s3AsyncClient, uploadRequest, returnFuture, uploadId)) + .exceptionally(throwable -> { + handleException(returnFuture, () -> "Unexpected exception occurred", throwable); + return null; + }); + } + + private void mergeAndVerifyChecksum( + AtomicReferenceArray inputStreamContainers, + String fileName, + long expectedChecksum + ) { + long resultantChecksum = fromBase64String(inputStreamContainers.get(0).getChecksum()); + for (int index = 1; index < inputStreamContainers.length(); index++) { + long curChecksum = fromBase64String(inputStreamContainers.get(index).getChecksum()); + resultantChecksum = JZlib.crc32_combine(resultantChecksum, curChecksum, inputStreamContainers.get(index).getContentLength()); + } + + if (resultantChecksum != expectedChecksum) { + throw new RuntimeException(new CorruptFileException("File level checksums didn't match combined part checksums", fileName)); + } + } + + private BiFunction handleExceptionOrResponse( + S3AsyncClient s3AsyncClient, + UploadRequest uploadRequest, + CompletableFuture returnFuture, + String uploadId + ) { + + return (response, throwable) -> { + if (throwable != null) { + AsyncPartsHandler.cleanUpParts(s3AsyncClient, uploadRequest, uploadId); + handleException(returnFuture, () -> "Failed to send multipart upload requests.", throwable); + } else { + returnFuture.complete(null); + } + + return null; + }; + } + + private CompletableFuture completeMultipartUpload( + S3AsyncClient s3AsyncClient, + UploadRequest uploadRequest, + String uploadId, + AtomicReferenceArray completedParts + ) { + + log.debug(() -> new ParameterizedMessage("Sending completeMultipartUploadRequest, uploadId: {}", uploadId)); + CompletedPart[] parts = IntStream.range(0, completedParts.length()).mapToObj(completedParts::get).toArray(CompletedPart[]::new); + CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder() + .bucket(uploadRequest.getBucket()) + .key(uploadRequest.getKey()) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(parts).build()) + .build(); + + return SocketAccess.doPrivileged(() -> s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest)); + } + + private static String base64StringFromLong(Long val) { + return Base64.getEncoder().encodeToString(Arrays.copyOfRange(ByteUtils.toByteArrayBE(val), 4, 8)); + } + + private static long fromBase64String(String base64String) { + byte[] decodedBytes = Base64.getDecoder().decode(base64String); + if (decodedBytes.length != 4) { + throw new IllegalArgumentException("Invalid Base64 encoded CRC32 checksum"); + } + long result = 0; + for (int i = 0; i < 4; i++) { + result <<= 8; + result |= (decodedBytes[i] & 0xFF); + } + return result; + } + + private static void handleException(CompletableFuture returnFuture, Supplier message, Throwable throwable) { + Throwable cause = throwable instanceof CompletionException ? throwable.getCause() : throwable; + + if (cause instanceof Error) { + returnFuture.completeExceptionally(cause); + } else { + SdkClientException exception = SdkClientException.create(message.get(), cause); + returnFuture.completeExceptionally(exception); + } + } + + /** + * Calculates the optimal part size of each part request if the upload operation is carried out as multipart upload. + */ + public long calculateOptimalPartSize(long contentLengthOfSource) { + if (contentLengthOfSource < ByteSizeUnit.MB.toBytes(5)) { + return contentLengthOfSource; + } + double optimalPartSize = contentLengthOfSource / (double) MAX_UPLOAD_PARTS; + optimalPartSize = Math.ceil(optimalPartSize); + return (long) Math.max(optimalPartSize, minimumPartSize); + } + + private void uploadInOneChunk( + S3AsyncClient s3AsyncClient, + UploadRequest uploadRequest, + InputStreamContainer inputStreamContainer, + CompletableFuture returnFuture + ) { + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() + .bucket(uploadRequest.getBucket()) + .key(uploadRequest.getKey()) + .contentLength(uploadRequest.getContentLength()); + if (uploadRequest.doRemoteDataIntegrityCheck()) { + putObjectRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32); + putObjectRequestBuilder.checksumCRC32(base64StringFromLong(uploadRequest.getExpectedChecksum())); + } + ExecutorService streamReadExecutor = uploadRequest.getWritePriority() == WritePriority.HIGH + ? priorityExecutorService + : executorService; + CompletableFuture putObjectFuture = SocketAccess.doPrivileged( + () -> s3AsyncClient.putObject( + putObjectRequestBuilder.build(), + AsyncRequestBody.fromInputStream( + inputStreamContainer.getInputStream(), + inputStreamContainer.getContentLength(), + streamReadExecutor + ) + ).handle((resp, throwable) -> { + if (throwable != null) { + Throwable unwrappedThrowable = ExceptionsHelper.unwrap(throwable, S3Exception.class); + if (unwrappedThrowable != null) { + S3Exception s3Exception = (S3Exception) unwrappedThrowable; + if (s3Exception.statusCode() == HttpStatusCode.BAD_REQUEST + && "BadDigest".equals(s3Exception.awsErrorDetails().errorCode())) { + throw new RuntimeException(new CorruptFileException(s3Exception, uploadRequest.getKey())); + } + } + returnFuture.completeExceptionally(throwable); + } else { + try { + uploadRequest.getUploadFinalizer().accept(true); + } catch (IOException e) { + throw new RuntimeException(e); + } + returnFuture.complete(null); + } + + return null; + }).handle((resp, throwable) -> { + if (throwable != null) { + deleteUploadedObject(s3AsyncClient, uploadRequest); + returnFuture.completeExceptionally(throwable); + } + + return null; + }) + ); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, putObjectFuture); + CompletableFutureUtils.forwardResultTo(putObjectFuture, returnFuture); + } + + private void deleteUploadedObject(S3AsyncClient s3AsyncClient, UploadRequest uploadRequest) { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(uploadRequest.getBucket()) + .key(uploadRequest.getKey()) + .build(); + + SocketAccess.doPrivileged(() -> s3AsyncClient.deleteObject(deleteObjectRequest)).exceptionally(throwable -> { + log.error(() -> new ParameterizedMessage("Failed to delete uploaded object of key {}", uploadRequest.getKey()), throwable); + return null; + }); + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java new file mode 100644 index 0000000000000..3804c8417eb9f --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import org.opensearch.common.CheckedConsumer; +import org.opensearch.common.blobstore.stream.write.WritePriority; + +import java.io.IOException; + +/** + * A model encapsulating all details for an upload to S3 + */ +public class UploadRequest { + private final String bucket; + private final String key; + private final long contentLength; + private final WritePriority writePriority; + private final CheckedConsumer uploadFinalizer; + private final boolean doRemoteDataIntegrityCheck; + private final Long expectedChecksum; + + /** + * Construct a new UploadRequest object + * + * @param bucket The name of the S3 bucket + * @param key Key of the file needed to be uploaded + * @param contentLength Total content length of the file for upload + * @param writePriority The priority of this upload + * @param uploadFinalizer An upload finalizer to call once all parts are uploaded + * @param doRemoteDataIntegrityCheck A boolean to inform vendor plugins whether remote data integrity checks need to be done + * @param expectedChecksum Checksum of the file being uploaded for remote data integrity check + */ + public UploadRequest( + String bucket, + String key, + long contentLength, + WritePriority writePriority, + CheckedConsumer uploadFinalizer, + boolean doRemoteDataIntegrityCheck, + Long expectedChecksum + ) { + this.bucket = bucket; + this.key = key; + this.contentLength = contentLength; + this.writePriority = writePriority; + this.uploadFinalizer = uploadFinalizer; + this.doRemoteDataIntegrityCheck = doRemoteDataIntegrityCheck; + this.expectedChecksum = expectedChecksum; + } + + public String getBucket() { + return bucket; + } + + public String getKey() { + return key; + } + + public long getContentLength() { + return contentLength; + } + + public WritePriority getWritePriority() { + return writePriority; + } + + public CheckedConsumer getUploadFinalizer() { + return uploadFinalizer; + } + + public boolean doRemoteDataIntegrityCheck() { + return doRemoteDataIntegrityCheck; + } + + public Long getExpectedChecksum() { + return expectedChecksum; + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/io/CheckedContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/io/CheckedContainer.java new file mode 100644 index 0000000000000..0596424093dca --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/io/CheckedContainer.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.io; + +public class CheckedContainer { + + private String checksum; + private long contentLength; + + public CheckedContainer(long contentLength) { + this.contentLength = contentLength; + } + + public void setChecksum(String checksum) { + this.checksum = checksum; + } + + public String getChecksum() { + return checksum; + } + + public long getContentLength() { + return contentLength; + } +} diff --git a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy index d74d3048addbf..106103d45e7eb 100644 --- a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy @@ -35,6 +35,7 @@ grant { // TODO: get these fixed in aws sdk permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "getClassLoader"; + permission java.lang.RuntimePermission "setContextClassLoader"; // Needed because of problems in AmazonS3Client: // When no region is set on a AmazonS3Client instance, the // AWS SDK loads all known partitions from a JSON file and @@ -56,10 +57,10 @@ grant { // only for tests : org.opensearch.repositories.s3.S3RepositoryPlugin permission java.util.PropertyPermission "opensearch.allow_insecure_settings", "read,write"; - permission java.util.PropertyPermission "aws.sharedCredentialsFile", "read,write"; permission java.util.PropertyPermission "aws.configFile", "read,write"; permission java.util.PropertyPermission "opensearch.path.conf", "read,write"; - permission java.io.FilePermission "config", "read"; + + permission java.lang.RuntimePermission "accessDeclaredMembers"; }; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java new file mode 100644 index 0000000000000..a401ba06728d7 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.junit.Before; +import org.opensearch.cli.SuppressForbidden; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.settings.MockSecureSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.concurrent.Executors; + +public class S3AsyncServiceTests extends OpenSearchTestCase implements ConfigPathSupport { + + @Override + @Before + @SuppressForbidden(reason = "Need to set opensearch.path.conf for async client") + public void setUp() throws Exception { + SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", configPath().toString())); + super.setUp(); + } + + public void testCachedClientsAreReleased() { + final S3AsyncService s3AsyncService = new S3AsyncService(configPath()); + final Settings settings = Settings.builder().put("endpoint", "http://first").put("region", "us-east-2").build(); + final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); + final RepositoryMetadata metadata2 = new RepositoryMetadata("second", "s3", settings); + final AsyncExecutorContainer asyncExecutorContainer = new AsyncExecutorContainer( + Executors.newSingleThreadExecutor(), + Executors.newSingleThreadExecutor(), + new AsyncTransferEventLoopGroup(1) + ); + final S3ClientSettings clientSettings = s3AsyncService.settings(metadata2); + final S3ClientSettings otherClientSettings = s3AsyncService.settings(metadata2); + assertSame(clientSettings, otherClientSettings); + final AmazonAsyncS3Reference reference = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer) + ); + reference.close(); + s3AsyncService.close(); + final AmazonAsyncS3Reference referenceReloaded = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer) + ); + assertNotSame(referenceReloaded, reference); + referenceReloaded.close(); + s3AsyncService.close(); + final S3ClientSettings clientSettingsReloaded = s3AsyncService.settings(metadata1); + assertNotSame(clientSettings, clientSettingsReloaded); + } + + public void testCachedClientsWithCredentialsAreReleased() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.role_arn", "role"); + final Map defaults = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build(), + configPath() + ); + final S3AsyncService s3AsyncService = new S3AsyncService(configPath()); + s3AsyncService.refreshAndClearCache(defaults); + final Settings settings = Settings.builder().put("endpoint", "http://first").put("region", "us-east-2").build(); + final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); + final RepositoryMetadata metadata2 = new RepositoryMetadata("second", "s3", settings); + final AsyncExecutorContainer asyncExecutorContainer = new AsyncExecutorContainer( + Executors.newSingleThreadExecutor(), + Executors.newSingleThreadExecutor(), + new AsyncTransferEventLoopGroup(1) + ); + final S3ClientSettings clientSettings = s3AsyncService.settings(metadata2); + final S3ClientSettings otherClientSettings = s3AsyncService.settings(metadata2); + assertSame(clientSettings, otherClientSettings); + final AmazonAsyncS3Reference reference = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer) + ); + reference.close(); + s3AsyncService.close(); + final AmazonAsyncS3Reference referenceReloaded = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer) + ); + assertNotSame(referenceReloaded, reference); + referenceReloaded.close(); + s3AsyncService.close(); + final S3ClientSettings clientSettingsReloaded = s3AsyncService.settings(metadata1); + assertNotSame(clientSettings, clientSettingsReloaded); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 9f5ebc5afe017..57d0387c96095 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -56,6 +56,7 @@ import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.io.SdkDigestInputStream; import software.amazon.awssdk.utils.internal.Base16; +import org.opensearch.repositories.blobstore.ZeroInputStream; import java.io.ByteArrayInputStream; import java.io.FilterInputStream; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java new file mode 100644 index 0000000000000..596291a1d94fb --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java @@ -0,0 +1,238 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import org.junit.Before; +import org.opensearch.ExceptionsHelper; +import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.exception.CorruptFileException; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.repositories.blobstore.ZeroInputStream; +import org.opensearch.test.OpenSearchTestCase; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.http.HttpStatusCode; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AsyncTransferManagerTests extends OpenSearchTestCase { + + private AsyncTransferManager asyncTransferManager; + private S3AsyncClient s3AsyncClient; + + @Override + @Before + public void setUp() throws Exception { + s3AsyncClient = mock(S3AsyncClient.class); + asyncTransferManager = new AsyncTransferManager( + ByteSizeUnit.MB.toBytes(5), + Executors.newSingleThreadExecutor(), + Executors.newSingleThreadExecutor() + ); + super.setUp(); + } + + public void testOneChunkUpload() { + CompletableFuture putObjectResponseCompletableFuture = new CompletableFuture<>(); + putObjectResponseCompletableFuture.complete(PutObjectResponse.builder().build()); + when(s3AsyncClient.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))).thenReturn( + putObjectResponseCompletableFuture + ); + + CompletableFuture resultFuture = asyncTransferManager.uploadObject( + s3AsyncClient, + new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(1), WritePriority.HIGH, uploadSuccess -> { + // do nothing + }, false, null), + new StreamContext( + (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position), + ByteSizeUnit.MB.toBytes(1), + ByteSizeUnit.MB.toBytes(1), + 1 + ) + ); + + try { + resultFuture.get(); + } catch (ExecutionException | InterruptedException e) { + fail("did not expect resultFuture to fail"); + } + + verify(s3AsyncClient, times(1)).putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)); + } + + public void testOneChunkUploadCorruption() { + CompletableFuture putObjectResponseCompletableFuture = new CompletableFuture<>(); + putObjectResponseCompletableFuture.completeExceptionally( + S3Exception.builder() + .statusCode(HttpStatusCode.BAD_REQUEST) + .awsErrorDetails(AwsErrorDetails.builder().errorCode("BadDigest").build()) + .build() + ); + when(s3AsyncClient.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))).thenReturn( + putObjectResponseCompletableFuture + ); + + CompletableFuture deleteObjectResponseCompletableFuture = new CompletableFuture<>(); + deleteObjectResponseCompletableFuture.complete(DeleteObjectResponse.builder().build()); + when(s3AsyncClient.deleteObject(any(DeleteObjectRequest.class))).thenReturn(deleteObjectResponseCompletableFuture); + + CompletableFuture resultFuture = asyncTransferManager.uploadObject( + s3AsyncClient, + new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(1), WritePriority.HIGH, uploadSuccess -> { + // do nothing + }, false, null), + new StreamContext( + (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position), + ByteSizeUnit.MB.toBytes(1), + ByteSizeUnit.MB.toBytes(1), + 1 + ) + ); + + try { + resultFuture.get(); + fail("did not expect resultFuture to pass"); + } catch (ExecutionException | InterruptedException e) { + Throwable throwable = ExceptionsHelper.unwrap(e, CorruptFileException.class); + assertNotNull(throwable); + assertTrue(throwable instanceof CorruptFileException); + } + + verify(s3AsyncClient, times(1)).putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)); + verify(s3AsyncClient, times(1)).deleteObject(any(DeleteObjectRequest.class)); + } + + public void testMultipartUpload() { + CompletableFuture createMultipartUploadRequestCompletableFuture = new CompletableFuture<>(); + createMultipartUploadRequestCompletableFuture.complete(CreateMultipartUploadResponse.builder().uploadId("uploadId").build()); + when(s3AsyncClient.createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn( + createMultipartUploadRequestCompletableFuture + ); + + CompletableFuture uploadPartResponseCompletableFuture = new CompletableFuture<>(); + uploadPartResponseCompletableFuture.complete(UploadPartResponse.builder().checksumCRC32("pzjqHA==").build()); + when(s3AsyncClient.uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class))).thenReturn( + uploadPartResponseCompletableFuture + ); + + CompletableFuture completeMultipartUploadResponseCompletableFuture = new CompletableFuture<>(); + completeMultipartUploadResponseCompletableFuture.complete(CompleteMultipartUploadResponse.builder().build()); + when(s3AsyncClient.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenReturn( + completeMultipartUploadResponseCompletableFuture + ); + + CompletableFuture abortMultipartUploadResponseCompletableFuture = new CompletableFuture<>(); + abortMultipartUploadResponseCompletableFuture.complete(AbortMultipartUploadResponse.builder().build()); + when(s3AsyncClient.abortMultipartUpload(any(AbortMultipartUploadRequest.class))).thenReturn( + abortMultipartUploadResponseCompletableFuture + ); + + CompletableFuture resultFuture = asyncTransferManager.uploadObject( + s3AsyncClient, + new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(5), WritePriority.HIGH, uploadSuccess -> { + // do nothing + }, true, 3376132981L), + new StreamContext( + (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position), + ByteSizeUnit.MB.toBytes(1), + ByteSizeUnit.MB.toBytes(1), + 5 + ) + ); + + try { + resultFuture.get(); + } catch (ExecutionException | InterruptedException e) { + fail("did not expect resultFuture to fail"); + } + + verify(s3AsyncClient, times(1)).createMultipartUpload(any(CreateMultipartUploadRequest.class)); + verify(s3AsyncClient, times(5)).uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class)); + verify(s3AsyncClient, times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + verify(s3AsyncClient, times(0)).abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + } + + public void testMultipartUploadCorruption() { + CompletableFuture createMultipartUploadRequestCompletableFuture = new CompletableFuture<>(); + createMultipartUploadRequestCompletableFuture.complete(CreateMultipartUploadResponse.builder().uploadId("uploadId").build()); + when(s3AsyncClient.createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn( + createMultipartUploadRequestCompletableFuture + ); + + CompletableFuture uploadPartResponseCompletableFuture = new CompletableFuture<>(); + uploadPartResponseCompletableFuture.complete(UploadPartResponse.builder().checksumCRC32("pzjqHA==").build()); + when(s3AsyncClient.uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class))).thenReturn( + uploadPartResponseCompletableFuture + ); + + CompletableFuture completeMultipartUploadResponseCompletableFuture = new CompletableFuture<>(); + completeMultipartUploadResponseCompletableFuture.complete(CompleteMultipartUploadResponse.builder().build()); + when(s3AsyncClient.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenReturn( + completeMultipartUploadResponseCompletableFuture + ); + + CompletableFuture abortMultipartUploadResponseCompletableFuture = new CompletableFuture<>(); + abortMultipartUploadResponseCompletableFuture.complete(AbortMultipartUploadResponse.builder().build()); + when(s3AsyncClient.abortMultipartUpload(any(AbortMultipartUploadRequest.class))).thenReturn( + abortMultipartUploadResponseCompletableFuture + ); + + CompletableFuture resultFuture = asyncTransferManager.uploadObject( + s3AsyncClient, + new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(5), WritePriority.HIGH, uploadSuccess -> { + // do nothing + }, true, 0L), + new StreamContext( + (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position), + ByteSizeUnit.MB.toBytes(1), + ByteSizeUnit.MB.toBytes(1), + 5 + ) + ); + + try { + resultFuture.get(); + fail("did not expect resultFuture to pass"); + } catch (ExecutionException | InterruptedException e) { + Throwable throwable = ExceptionsHelper.unwrap(e, CorruptFileException.class); + assertNotNull(throwable); + assertTrue(throwable instanceof CorruptFileException); + } + + verify(s3AsyncClient, times(1)).createMultipartUpload(any(CreateMultipartUploadRequest.class)); + verify(s3AsyncClient, times(5)).uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class)); + verify(s3AsyncClient, times(0)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + verify(s3AsyncClient, times(1)).abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + } +} diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index adaf95ae67a8e..0361652eb2457 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -58,8 +58,6 @@ import java.util.Arrays; import java.util.Locale; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -409,85 +407,4 @@ protected void sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws exchange.getResponseBody().flush(); } } - - /** - * A resettable InputStream that only serves zeros. - **/ - public static class ZeroInputStream extends InputStream { - - private final AtomicBoolean closed = new AtomicBoolean(false); - private final long length; - private final AtomicLong reads; - private volatile long mark; - - public ZeroInputStream(final long length) { - this.length = length; - this.reads = new AtomicLong(0); - this.mark = -1; - } - - @Override - public int read() throws IOException { - ensureOpen(); - return (reads.incrementAndGet() <= length) ? 0 : -1; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - ensureOpen(); - if (len == 0) { - return 0; - } - - final int available = available(); - if (available == 0) { - return -1; - } - - final int toCopy = Math.min(len, available); - Arrays.fill(b, off, off + toCopy, (byte) 0); - reads.addAndGet(toCopy); - return toCopy; - } - - @Override - public boolean markSupported() { - return true; - } - - @Override - public synchronized void mark(int readlimit) { - mark = reads.get(); - } - - @Override - public synchronized void reset() throws IOException { - ensureOpen(); - reads.set(mark); - } - - @Override - public int available() throws IOException { - ensureOpen(); - if (reads.get() >= length) { - return 0; - } - try { - return Math.toIntExact(length - reads.get()); - } catch (ArithmeticException e) { - return Integer.MAX_VALUE; - } - } - - @Override - public void close() { - closed.set(true); - } - - private void ensureOpen() throws IOException { - if (closed.get()) { - throw new IOException("Stream closed"); - } - } - } } diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/ZeroInputStream.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/ZeroInputStream.java new file mode 100644 index 0000000000000..f299c17f16a06 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/ZeroInputStream.java @@ -0,0 +1,96 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.blobstore; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A resettable InputStream that only serves zeros. + **/ +public class ZeroInputStream extends InputStream { + + private final AtomicBoolean closed = new AtomicBoolean(false); + private final long length; + private final AtomicLong reads; + private volatile long mark; + + public ZeroInputStream(final long length) { + this.length = length; + this.reads = new AtomicLong(0); + this.mark = -1; + } + + @Override + public int read() throws IOException { + ensureOpen(); + return (reads.incrementAndGet() <= length) ? 0 : -1; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + if (len == 0) { + return 0; + } + + final int available = available(); + if (available == 0) { + return -1; + } + + final int toCopy = Math.min(len, available); + Arrays.fill(b, off, off + toCopy, (byte) 0); + reads.addAndGet(toCopy); + return toCopy; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void mark(int readlimit) { + mark = reads.get(); + } + + @Override + public synchronized void reset() throws IOException { + ensureOpen(); + reads.set(mark); + } + + @Override + public int available() throws IOException { + ensureOpen(); + if (reads.get() >= length) { + return 0; + } + try { + return Math.toIntExact(length - reads.get()); + } catch (ArithmeticException e) { + return Integer.MAX_VALUE; + } + } + + @Override + public void close() { + closed.set(true); + } + + private void ensureOpen() throws IOException { + if (closed.get()) { + throw new IOException("Stream closed"); + } + } +} From e3542011cd9584c354405ba6cbf9a31ced7bc5ce Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Wed, 12 Jul 2023 16:29:27 -0500 Subject: [PATCH 115/186] [Refactor] StreamIO from common to core.common namespace in core lib (#8157) * [Refactor] StreamIO from common to core.common in core lib This PR is a mechanical refactor of all Stream classes in o.o.common.io.stream to o.o.core.common.io.stream namespace. In addition to the namespace refactor several loose end classes in o.o.common.io.stream of the :server module are refactored to o.o.core.common.io.stream of the :libs:opensearch-core library. These classes include the following: * ByteBufferStreamInpu * BytesStreamInput * DataOutputStreamOutput * FilterStreamInput * NamedWriteableAwareStreamInput * OutputStreamStreamOutput * VersionedNamedWriteable Companion tests are also refactored from the server test folder to the corresponding core test folder. Signed-off-by: Nicholas Walter Knize --- .../aggregations/TermsReduceBenchmark.java | 6 ++-- .../StringTermsSerializationBenchmark.java | 2 +- .../noop/action/bulk/RestNoopBulkAction.java | 4 +-- .../action/bulk/TransportNoopBulkAction.java | 2 +- .../search/TransportNoopSearchAction.java | 2 +- .../org/opensearch/client/ClusterClient.java | 2 +- .../opensearch/client/GetAliasesResponse.java | 4 +-- .../org/opensearch/client/IndicesClient.java | 2 +- .../opensearch/client/RequestConverters.java | 2 +- .../client/RestHighLevelClient.java | 2 +- .../client/cluster/RemoteInfoResponse.java | 2 +- .../client/core/BroadcastResponse.java | 2 +- .../opensearch/client/core/CountResponse.java | 4 +-- .../client/core/TermVectorsRequest.java | 2 +- .../client/indices/CloseIndexResponse.java | 4 +-- .../client/indices/CreateIndexRequest.java | 4 +-- .../indices/GetFieldMappingsResponse.java | 4 +-- .../client/indices/GetIndexResponse.java | 2 +- .../client/indices/GetMappingsResponse.java | 2 +- .../indices/PutIndexTemplateRequest.java | 4 +-- .../client/indices/PutMappingRequest.java | 4 +-- .../GetSnapshotLifecyclePolicyResponse.java | 2 +- .../client/tasks/OpenSearchException.java | 2 +- .../client/AbstractRequestTestCase.java | 2 +- .../client/AbstractResponseTestCase.java | 2 +- .../opensearch/client/BulkProcessorIT.java | 2 +- .../client/BulkProcessorRetryIT.java | 2 +- .../opensearch/client/ClusterClientIT.java | 2 +- .../java/org/opensearch/client/CrudIT.java | 4 +-- .../client/GetAliasesResponseTests.java | 2 +- .../opensearch/client/IndicesClientIT.java | 4 +-- .../org/opensearch/client/IngestClientIT.java | 2 +- .../client/IngestRequestConvertersTests.java | 2 +- ...OpenSearchRestHighLevelClientTestCase.java | 2 +- .../java/org/opensearch/client/ReindexIT.java | 2 +- .../client/RequestConvertersTests.java | 4 +-- .../client/RestHighLevelClientTests.java | 4 +-- .../java/org/opensearch/client/SearchIT.java | 4 +-- .../client/SearchPipelineClientIT.java | 2 +- .../org/opensearch/client/SnapshotIT.java | 2 +- .../opensearch/client/StoredScriptsIT.java | 4 +-- .../java/org/opensearch/client/TasksIT.java | 2 +- .../opensearch/client/UpdateByQueryIT.java | 2 +- .../client/core/BroadcastResponseTests.java | 2 +- .../client/core/CountResponseTests.java | 6 ++-- .../client/core/GetSourceResponseTests.java | 4 +-- .../core/tasks/GetTaskResponseTests.java | 2 +- .../documentation/CRUDDocumentationIT.java | 4 +-- .../ClusterClientDocumentationIT.java | 2 +- .../IndicesClientDocumentationIT.java | 4 +-- .../IngestClientDocumentationIT.java | 2 +- .../documentation/SearchDocumentationIT.java | 6 ++-- .../SnapshotClientDocumentationIT.java | 2 +- .../StoredScriptsDocumentationIT.java | 4 +-- .../indices/CloseIndexResponseTests.java | 6 ++-- .../DataStreamsStatsResponseTests.java | 2 +- .../indices/GetDataStreamResponseTests.java | 2 +- .../GetFieldMappingsResponseTests.java | 2 +- .../GetIndexTemplatesResponseTests.java | 4 +-- .../tasks/CancelTasksResponseTests.java | 4 +-- .../settings/CreateKeyStoreCommand.java | 1 + .../opensearch/bootstrap/BootstrapTests.java | 2 +- .../common/settings/KeyStoreWrapperTests.java | 1 + .../upgrade/ImportKeystoreTask.java | 2 +- .../java/org/opensearch/ExceptionsHelper.java | 8 +++--- .../org/opensearch/OpenSearchException.java | 28 +++++++++---------- .../opensearch/common/io/package-info.java | 10 ------- .../org/opensearch/common/package-info.java | 10 ------- .../common/xcontent/package-info.java | 10 ------- .../action/ShardOperationFailedException.java | 6 ++-- .../{ => core}/action/package-info.java | 2 +- .../DefaultShardOperationFailedException.java | 12 ++++---- .../action/support/package-info.java | 2 +- .../{ => core}/common/ParsingException.java | 8 +++--- .../common/bytes/AbstractBytesReference.java | 4 +-- .../{ => core}/common/bytes/BytesArray.java | 4 +-- .../common/bytes/BytesReference.java | 8 +++--- .../common/bytes/CompositeBytesReference.java | 2 +- .../common/bytes/PagedBytesReference.java | 4 +-- .../{ => core}/common/bytes/package-info.java | 2 +- .../common/compress/NotXContentException.java | 2 +- .../common/compress/package-info.java | 2 +- .../io/stream/ByteBufferStreamInput.java | 2 +- .../common/io/stream/BytesStream.java | 4 +-- .../common/io/stream/BytesStreamInput.java | 2 +- .../io/stream/DataOutputStreamOutput.java | 2 +- .../common/io/stream/FilterStreamInput.java | 2 +- .../io/stream/InputStreamStreamInput.java | 2 +- .../common/io/stream/NamedWriteable.java | 2 +- .../NamedWriteableAwareStreamInput.java | 2 +- .../io/stream/NamedWriteableRegistry.java | 4 +-- .../NotSerializableExceptionWrapper.java | 4 +-- .../io/stream/OutputStreamStreamOutput.java | 2 +- .../common/io/stream/StreamInput.java | 12 ++++---- .../common/io/stream/StreamOutput.java | 14 +++++----- .../io/stream/VersionedNamedWriteable.java | 2 +- .../common/io/stream/Writeable.java | 2 +- .../common/io/stream/package-info.java | 2 +- .../common/settings/SecureString.java | 2 +- .../common/settings/package-info.java | 2 +- .../{ => core}/common/text/Text.java | 6 ++-- .../{ => core}/common/text/package-info.java | 2 +- .../{ => core}/common/util/BigArray.java | 2 +- .../{ => core}/common/util/ByteArray.java | 2 +- .../{ => core}/common/util/package-info.java | 2 +- .../opensearch/{ => core}/index/Index.java | 8 +++--- .../{ => core}/index/package-info.java | 2 +- .../{ => core}/index/shard/ShardId.java | 10 +++---- .../{ => core}/index/shard/package-info.java | 2 +- .../IndexShardSnapshotException.java | 6 ++-- .../IndexShardSnapshotFailedException.java | 6 ++-- .../index/snapshots/package-info.java | 2 +- .../{ => core}/rest/RestStatus.java | 8 +++--- .../{ => core}/rest/package-info.java | 2 +- .../xcontent/XContentParserUtils.java | 9 ++---- ...ultShardOperationFailedExceptionTests.java | 12 ++++---- .../core}/common/io/StreamsTests.java | 7 +++-- .../common/io/stream/BaseStreamTests.java | 10 ++++--- .../io/stream/ByteBufferStreamInputTests.java | 4 +-- .../BytesReferenceStreamInputTests.java | 4 +-- .../io/stream/BytesStreamInputTests.java | 4 +-- .../io/stream/FilterStreamInputTests.java | 4 +-- .../stream/InputStreamStreamInputTests.java | 4 +-- .../stream/NamedWriteableRegistryTests.java | 2 +- .../core/util/BytesRefUtilsTests.java | 2 +- .../ssl/SslConfigurationLoaderTests.java | 2 +- .../xcontent/MapXContentParserTests.java | 2 +- .../common/xcontent/ObjectParserTests.java | 1 + .../common/xcontent/XContentParserTests.java | 2 +- .../matrix/stats/InternalMatrixStats.java | 4 +-- .../stats/MatrixStatsAggregationBuilder.java | 4 +-- .../matrix/stats/MatrixStatsResults.java | 6 ++-- .../matrix/stats/RunningStats.java | 6 ++-- .../ArrayValuesSourceAggregationBuilder.java | 4 +-- .../support/ArrayValuesSourceParser.java | 2 +- .../common/CommonAnalysisModulePlugin.java | 2 +- .../CharGroupTokenizerFactoryTests.java | 2 +- .../common/EdgeNGramTokenizerTests.java | 2 +- .../FlattenGraphTokenFilterFactoryTests.java | 2 +- .../common/NGramTokenizerFactoryTests.java | 2 +- .../PathHierarchyTokenizerFactoryTests.java | 2 +- .../WhitespaceTokenizerFactoryTests.java | 2 +- .../metrics/GeoBoundsITTestCase.java | 2 +- .../GeoTileGridValuesSourceBuilder.java | 4 +-- .../bucket/geogrid/BaseGeoGrid.java | 4 +-- .../bucket/geogrid/BaseGeoGridBucket.java | 4 +-- .../geogrid/GeoGridAggregationBuilder.java | 4 +-- .../bucket/geogrid/GeoHashGrid.java | 2 +- .../GeoHashGridAggregationBuilder.java | 2 +- .../bucket/geogrid/GeoTileGrid.java | 2 +- .../GeoTileGridAggregationBuilder.java | 2 +- .../geogrid/InternalGeoHashGridBucket.java | 2 +- .../geogrid/InternalGeoTileGridBucket.java | 2 +- .../metrics/GeoBoundsAggregationBuilder.java | 4 +-- .../metrics/InternalGeoBounds.java | 4 +-- .../ingest/common/IngestRestartIT.java | 4 +-- .../ingest/common/GrokProcessorGetAction.java | 4 +-- .../ingest/common/JsonProcessor.java | 4 +-- .../ingest/common/ScriptProcessor.java | 2 +- .../common/GrokProcessorGetActionTests.java | 4 +-- .../ingest/common/JsonProcessorTests.java | 2 +- .../geoip/GeoIpProcessorNonIngestNodeIT.java | 4 +-- .../script/expression/StoredExpressionIT.java | 2 +- .../script/mustache/SearchTemplateIT.java | 2 +- .../mustache/MultiSearchTemplateRequest.java | 6 ++-- .../mustache/MultiSearchTemplateResponse.java | 6 ++-- .../mustache/SearchTemplateRequest.java | 6 ++-- .../mustache/SearchTemplateResponse.java | 8 +++--- .../TransportSearchTemplateAction.java | 2 +- .../MultiSearchTemplateRequestTests.java | 2 +- .../script/mustache/MustacheTests.java | 2 +- .../mustache/SearchTemplateRequestTests.java | 2 +- .../SearchTemplateRequestXContentTests.java | 2 +- .../mustache/SearchTemplateResponseTests.java | 2 +- .../painless/PainlessModulePlugin.java | 2 +- .../action/PainlessContextAction.java | 6 ++-- .../PainlessContextClassBindingInfo.java | 6 ++-- .../action/PainlessContextClassInfo.java | 6 ++-- .../PainlessContextConstructorInfo.java | 6 ++-- .../action/PainlessContextFieldInfo.java | 6 ++-- .../painless/action/PainlessContextInfo.java | 6 ++-- .../PainlessContextInstanceBindingInfo.java | 6 ++-- .../action/PainlessContextMethodInfo.java | 6 ++-- .../action/PainlessExecuteAction.java | 12 ++++---- .../org/opensearch/painless/DebugTests.java | 2 +- .../painless/action/ContextInfoTests.java | 2 +- .../action/PainlessExecuteApiTests.java | 2 +- .../action/PainlessExecuteRequestTests.java | 8 +++--- .../action/PainlessExecuteResponseTests.java | 2 +- .../index/query/RankFeatureQueryBuilder.java | 4 +-- .../RankFeatureMetaFieldMapperTests.java | 2 +- .../mapper/ScaledFloatFieldMapperTests.java | 2 +- .../join/query/ChildQuerySearchIT.java | 2 +- .../ChildrenAggregationBuilder.java | 6 ++-- .../join/aggregations/InternalChildren.java | 2 +- .../join/aggregations/InternalParent.java | 2 +- .../ParentAggregationBuilder.java | 6 ++-- .../join/query/HasChildQueryBuilder.java | 6 ++-- .../join/query/HasParentQueryBuilder.java | 6 ++-- .../join/query/ParentIdQueryBuilder.java | 6 ++-- .../ChildrenToParentAggregatorTests.java | 4 +-- .../ParentToChildrenAggregatorTests.java | 4 +-- .../mapper/ParentJoinFieldMapperTests.java | 2 +- .../percolator/PercolatorQuerySearchIT.java | 4 +-- .../opensearch/percolator/PercolateQuery.java | 2 +- .../percolator/PercolateQueryBuilder.java | 12 ++++---- .../percolator/PercolatorFieldMapper.java | 6 ++-- .../PercolatorHighlightSubFetchPhase.java | 2 +- .../percolator/CandidateQueryTests.java | 4 +-- .../PercolateQueryBuilderTests.java | 4 +-- .../percolator/PercolateQueryTests.java | 2 +- .../PercolateWithNestedQueryBuilderTests.java | 2 +- .../PercolatorFieldMapperTests.java | 12 ++++---- ...PercolatorHighlightSubFetchPhaseTests.java | 2 +- .../PercolatorQuerySearchTests.java | 2 +- .../percolator/QueryBuilderStoreTests.java | 2 +- .../rankeval/DiscountedCumulativeGain.java | 4 +-- .../index/rankeval/EvalQueryQuality.java | 8 +++--- .../index/rankeval/EvaluationMetric.java | 2 +- .../rankeval/ExpectedReciprocalRank.java | 4 +-- .../index/rankeval/MeanReciprocalRank.java | 4 +-- .../index/rankeval/MetricDetail.java | 2 +- .../index/rankeval/PrecisionAtK.java | 4 +-- .../index/rankeval/RankEvalModulePlugin.java | 2 +- .../index/rankeval/RankEvalRequest.java | 4 +-- .../index/rankeval/RankEvalResponse.java | 6 ++-- .../index/rankeval/RankEvalSpec.java | 10 +++---- .../index/rankeval/RatedDocument.java | 6 ++-- .../index/rankeval/RatedRequest.java | 6 ++-- .../index/rankeval/RatedSearchHit.java | 6 ++-- .../opensearch/index/rankeval/RecallAtK.java | 4 +-- .../rankeval/TransportRankEvalAction.java | 4 +-- .../DiscountedCumulativeGainTests.java | 6 ++-- .../index/rankeval/EvalQueryQualityTests.java | 8 +++--- .../rankeval/ExpectedReciprocalRankTests.java | 6 ++-- .../rankeval/MeanReciprocalRankTests.java | 6 ++-- .../index/rankeval/PrecisionAtKTests.java | 6 ++-- .../index/rankeval/RankEvalRequestTests.java | 4 +-- .../index/rankeval/RankEvalResponseTests.java | 8 +++--- .../index/rankeval/RankEvalSpecTests.java | 4 +-- .../index/rankeval/RatedDocumentTests.java | 4 +-- .../index/rankeval/RatedRequestsTests.java | 4 +-- .../index/rankeval/RatedSearchHitTests.java | 4 +-- .../index/rankeval/RecallAtKTests.java | 6 ++-- .../documentation/ReindexDocumentationIT.java | 2 +- .../AbstractAsyncBulkByScrollAction.java | 2 +- .../AbstractBaseReindexRestHandler.java | 2 +- .../AbstractBulkByQueryRestHandler.java | 4 +-- .../BulkByScrollParallelizationHelper.java | 2 +- ...kIndexByScrollResponseContentListener.java | 2 +- .../index/reindex/ReindexModulePlugin.java | 2 +- .../index/reindex/ReindexSslConfig.java | 2 +- .../opensearch/index/reindex/Reindexer.java | 2 +- .../reindex/RestDeleteByQueryAction.java | 2 +- .../index/reindex/RestReindexAction.java | 2 +- .../reindex/RestUpdateByQueryAction.java | 2 +- .../index/reindex/RethrottleRequest.java | 4 +-- .../reindex/TransportDeleteByQueryAction.java | 2 +- .../reindex/TransportUpdateByQueryAction.java | 2 +- .../reindex/remote/RemoteRequestBuilders.java | 2 +- .../reindex/remote/RemoteResponseParsers.java | 4 +-- .../remote/RemoteScrollableHitSource.java | 4 +-- .../reindex/AsyncBulkByScrollActionTests.java | 8 +++--- .../opensearch/index/reindex/CancelTests.java | 6 ++-- .../ClientScrollableHitSourceTests.java | 2 +- .../ReindexFromRemoteAllowlistTests.java | 4 +-- ...ReindexFromRemoteBuildRestClientTests.java | 4 +-- .../ReindexFromRemoteWithAuthTests.java | 6 ++-- .../reindex/ReindexRestClientSslTests.java | 2 +- .../ReindexSourceTargetValidationTests.java | 4 +-- .../index/reindex/RestReindexActionTests.java | 6 ++-- .../opensearch/index/reindex/RetryTests.java | 2 +- .../index/reindex/RoundTripTests.java | 8 +++--- .../index/reindex/remote/RemoteInfoTests.java | 2 +- .../remote/RemoteRequestBuildersTests.java | 4 +-- .../RemoteScrollableHitSourceTests.java | 6 ++-- .../repositories/url/URLFixture.java | 2 +- .../RepositoryURLClientYamlTestSuiteIT.java | 2 +- .../common/SearchPipelineCommonIT.java | 4 +-- .../common/FilterQueryRequestProcessor.java | 2 +- .../common/RenameFieldResponseProcessor.java | 2 +- .../common/ScriptRequestProcessor.java | 2 +- .../RenameFieldResponseProcessorTests.java | 2 +- .../systemd/SystemdModulePlugin.java | 2 +- .../opensearch/rest/Netty4BadRequestIT.java | 2 +- .../rest/Netty4HeadBodyIsEmptyIT.java | 4 +-- .../http/netty4/Netty4HttpRequest.java | 4 +-- .../http/netty4/Netty4HttpResponse.java | 4 +-- .../transport/Netty4ModulePlugin.java | 2 +- .../netty4/Netty4MessageChannelHandler.java | 2 +- .../transport/netty4/Netty4TcpChannel.java | 2 +- .../transport/netty4/Netty4Transport.java | 2 +- .../transport/netty4/Netty4Utils.java | 4 +-- .../http/netty4/Netty4BadRequestTests.java | 2 +- .../Netty4HttpPipeliningHandlerTests.java | 6 ++-- .../Netty4HttpServerPipeliningTests.java | 4 +-- .../Netty4HttpServerTransportTests.java | 6 ++-- .../Netty4SizeHeaderFrameDecoderTests.java | 2 +- .../transport/netty4/Netty4UtilsTests.java | 4 +-- .../netty4/NettyTransportMultiPortTests.java | 2 +- .../netty4/SimpleNetty4TransportTests.java | 2 +- .../ICUCollationKeywordFieldMapper.java | 2 +- .../analysis/icu/AnalysisICUPlugin.java | 2 +- .../analysis/IcuTokenizerFactoryTests.java | 2 +- .../analysis/SimpleIcuAnalysisTests.java | 2 +- .../SimpleIcuCollationTokenFilterTests.java | 2 +- .../SimpleIcuNormalizerCharFilterTests.java | 2 +- .../index/analysis/KuromojiAnalysisTests.java | 2 +- .../analysis/SimplePhoneticAnalysisTests.java | 2 +- .../SimpleSmartChineseAnalysisTests.java | 2 +- .../index/analysis/PolishAnalysisTests.java | 2 +- .../SimplePolishTokenFilterTests.java | 2 +- .../SimpleUkrainianAnalyzerTests.java | 2 +- .../analysis/UkrainianAnalysisTests.java | 2 +- .../discovery/ec2/AmazonEC2Fixture.java | 2 +- .../discovery/ec2/Ec2ClientSettings.java | 2 +- .../discovery/ec2/Ec2NetworkTests.java | 2 +- .../discovery/ec2/Ec2RetriesTests.java | 2 +- .../org/opensearch/cloud/gce/GCEFixture.java | 2 +- .../EventsCorrelationPluginTransportIT.java | 2 +- .../CorrelationVectorsEngineIT.java | 2 +- .../correlation/EventsCorrelationPlugin.java | 2 +- .../core/index/CorrelationParamsContext.java | 6 ++-- .../index/query/CorrelationQueryBuilder.java | 6 ++-- .../action/IndexCorrelationRuleRequest.java | 4 +-- .../action/IndexCorrelationRuleResponse.java | 6 ++-- .../rules/model/CorrelationQuery.java | 6 ++-- .../rules/model/CorrelationRule.java | 6 ++-- .../RestIndexCorrelationRuleAction.java | 2 +- .../TransportIndexCorrelationRuleAction.java | 2 +- .../index/CorrelationParamsContextTests.java | 2 +- .../core/index/VectorFieldTests.java | 2 +- .../CorrelationVectorFieldMapperTests.java | 2 +- .../query/CorrelationQueryBuilderTests.java | 8 +++--- .../ExampleCustomSettingsConfig.java | 2 +- .../customsigheuristic/SimpleHeuristic.java | 4 +-- .../SimpleHeuristicWireTests.java | 2 +- .../customsuggester/CustomSuggester.java | 2 +- .../customsuggester/CustomSuggestion.java | 6 ++-- .../CustomSuggestionBuilder.java | 6 ++-- .../rescore/ExampleRescoreBuilder.java | 4 +-- .../rescore/ExampleRescoreBuilderTests.java | 2 +- .../index/mapper/size/SizeMappingTests.java | 2 +- .../azure/AzureBlobStoreRepositoryTests.java | 2 +- .../azure/AzureStorageSettings.java | 2 +- .../azure/AzureBlobContainerRetriesTests.java | 4 +-- ...eCloudStorageBlobStoreRepositoryTests.java | 2 +- .../gcs/GoogleCloudStorageClientSettings.java | 2 +- ...CloudStorageBlobContainerRetriesTests.java | 6 ++-- .../gcs/GoogleCloudStorageServiceTests.java | 2 +- .../hdfs/HdfsBlobStoreContainerTests.java | 2 +- .../repositories/s3/S3ClientSettings.java | 2 +- .../repositories/s3/S3Repository.java | 2 +- .../s3/S3BlobContainerRetriesTests.java | 2 +- .../org/opensearch/http/nio/ByteBufUtils.java | 4 +-- .../opensearch/http/nio/NioHttpRequest.java | 4 +-- .../opensearch/http/nio/NioHttpResponse.java | 4 +-- .../transport/nio/NioTcpChannel.java | 2 +- .../transport/nio/NioTransport.java | 2 +- .../transport/nio/NioTransportPlugin.java | 2 +- .../transport/nio/TcpReadWriteHandler.java | 4 +-- .../http/nio/HttpReadWriteHandlerTests.java | 4 +-- .../nio/NioHttpPipeliningHandlerTests.java | 6 ++-- .../http/nio/NioHttpServerTransportTests.java | 6 ++-- .../nio/SimpleNioTransportTests.java | 2 +- .../upgrades/QueryBuilderBWCIT.java | 8 +++--- .../common/logging/JsonLoggerTests.java | 2 +- .../org/opensearch/backwards/IndexingIT.java | 2 +- .../org/opensearch/backwards/SearchingIT.java | 18 ------------ .../java/org/opensearch/search/CCSDuelIT.java | 2 +- .../org/opensearch/upgrades/RecoveryIT.java | 2 +- .../AwarenessAttributeDecommissionRestIT.java | 2 +- .../http/DanglingIndicesRestIT.java | 4 +-- .../http/IdentityAuthenticationIT.java | 2 +- .../http/IndexingPressureRestIT.java | 6 ++-- .../http/ShardIndexingPressureRestIT.java | 6 ++-- .../http/TestResponseHeaderRestAction.java | 2 +- .../opensearch/action/IndicesRequestIT.java | 2 +- .../action/admin/ReloadSecureSettingsIT.java | 2 +- .../ClusterAllocationExplainIT.java | 2 +- .../node/tasks/CancellableTasksIT.java | 4 +-- .../cluster/snapshots/SnapshotBlocksIT.java | 2 +- .../ValidateIndicesAliasesRequestIT.java | 2 +- .../admin/indices/create/ShrinkIndexIT.java | 2 +- .../admin/indices/create/SplitIndexIT.java | 2 +- .../datastream/DataStreamRolloverIT.java | 2 +- .../datastream/DataStreamTestCase.java | 2 +- .../indices/datastream/DataStreamUsageIT.java | 2 +- .../indices/forcemerge/ForceMergeIT.java | 2 +- .../put/ValidateMappingRequestPluginIT.java | 2 +- .../shards/IndicesShardStoreRequestIT.java | 2 +- .../action/bulk/BulkIntegrationIT.java | 4 +-- .../action/bulk/BulkProcessorRetryIT.java | 2 +- .../action/ingest/AsyncIngestProcessorIT.java | 6 ++-- .../action/search/TransportSearchIT.java | 6 ++-- .../support/WaitActiveShardCountIT.java | 2 +- ...tReplicationActionRetryOnClosedNodeIT.java | 6 ++-- .../cluster/ClusterStateDiffIT.java | 12 ++++---- .../cluster/NoClusterManagerNodeIT.java | 2 +- .../cluster/SimpleClusterStateIT.java | 6 ++-- .../cluster/allocation/ClusterRerouteIT.java | 4 +-- .../coordination/RareClusterStateIT.java | 2 +- .../metadata/TemplateUpgradeServiceIT.java | 2 +- .../cluster/routing/AllocationIdIT.java | 2 +- .../cluster/routing/PrimaryAllocationIT.java | 2 +- .../cluster/routing/WeightedRoutingIT.java | 2 +- .../common/settings/ConsistentSettingsIT.java | 1 + .../opensearch/explain/ExplainActionIT.java | 4 +-- .../opensearch/gateway/MetadataNodesIT.java | 2 +- .../gateway/RecoveryFromGatewayIT.java | 4 +-- .../gateway/ReplicaShardAllocatorIT.java | 2 +- .../java/org/opensearch/get/GetActionIT.java | 6 ++-- .../org/opensearch/index/FinalPipelineIT.java | 8 +++--- .../index/IndexRequestBuilderIT.java | 4 +-- .../index/SegmentReplicationPressureIT.java | 2 +- .../index/ShardIndexingPressureIT.java | 3 +- .../ShardIndexingPressureSettingsIT.java | 3 +- .../opensearch/index/WaitUntilRefreshIT.java | 2 +- .../index/engine/MaxDocsLimitIT.java | 2 +- .../index/seqno/RetentionLeaseIT.java | 2 +- .../opensearch/index/shard/IndexShardIT.java | 5 ++-- .../RemoveCorruptedShardDataCommandIT.java | 3 +- .../index/store/CorruptedFileIT.java | 6 ++-- .../indices/IndicesLifecycleListenerIT.java | 4 +-- .../breaker/CircuitBreakerServiceIT.java | 2 +- .../indices/recovery/DanglingIndicesIT.java | 2 +- .../indices/recovery/IndexRecoveryIT.java | 4 +-- .../replication/SegmentReplicationBaseIT.java | 4 +-- .../SegmentReplicationClusterSettingIT.java | 2 +- .../replication/SegmentReplicationIT.java | 4 +-- .../state/CloseWhileRelocatingShardsIT.java | 2 +- .../indices/state/OpenCloseIndexIT.java | 2 +- .../indices/stats/IndexStatsIT.java | 7 +++-- .../store/IndicesStoreIntegrationIT.java | 4 +-- .../template/SimpleIndexTemplateIT.java | 4 +-- .../org/opensearch/ingest/IngestClientIT.java | 2 +- ...gestProcessorNotInstalledOnAllNodesIT.java | 2 +- .../org/opensearch/mget/SimpleMgetIT.java | 2 +- .../recovery/RecoveryWhileUnderLoadIT.java | 2 +- .../org/opensearch/recovery/RelocationIT.java | 2 +- .../RemoteStoreBackpressureIT.java | 4 +-- .../fs/FsBlobStoreRepositoryIT.java | 2 +- .../opensearch/routing/SimpleRoutingIT.java | 4 +-- .../org/opensearch/script/ScriptCacheIT.java | 2 +- .../opensearch/script/StoredScriptsIT.java | 2 +- .../search/SearchWeightedRoutingIT.java | 4 +-- .../aggregations/FiltersAggsRewriteIT.java | 2 +- .../search/aggregations/bucket/NestedIT.java | 2 +- .../metrics/ScriptedMetricIT.java | 2 +- .../aggregations/pipeline/BucketScriptIT.java | 2 +- .../pipeline/BucketSelectorIT.java | 2 +- .../aggregations/pipeline/MaxBucketIT.java | 2 +- .../backpressure/SearchBackpressureIT.java | 4 +-- .../search/basic/SearchRedStateIndexIT.java | 2 +- .../search/fetch/FetchSubPhasePluginIT.java | 6 ++-- .../fetch/subphase/MatchedQueriesIT.java | 2 +- .../highlight/HighlighterSearchIT.java | 2 +- .../search/fields/SearchFieldsIT.java | 6 ++-- .../functionscore/FunctionScorePluginIT.java | 4 +-- .../opensearch/search/geo/GeoFilterIT.java | 2 +- .../search/preference/SearchPreferenceIT.java | 2 +- .../search/query/SearchQueryIT.java | 2 +- .../search/scroll/SearchScrollIT.java | 4 +-- .../search/simple/SimpleSearchIT.java | 2 +- .../opensearch/search/sort/FieldSortIT.java | 2 +- .../ContextCompletionSuggestSearchIT.java | 2 +- .../DedicatedClusterSnapshotRestoreIT.java | 10 +++---- ...etadataLoadingDuringSnapshotRestoreIT.java | 2 +- .../snapshots/RestoreSnapshotIT.java | 3 +- .../snapshots/SearchableSnapshotIT.java | 2 +- .../SegmentReplicationSnapshotIT.java | 4 +-- .../SharedClusterSnapshotRestoreIT.java | 4 +-- .../SnapshotCustomPluginStateIT.java | 4 +-- .../java/org/opensearch/update/UpdateIT.java | 2 +- .../org/opensearch/update/UpdateNoopIT.java | 2 +- .../validate/SimpleValidateQueryIT.java | 2 +- .../ConcurrentSeqNoVersioningIT.java | 14 +++++----- .../queries/BinaryDocValuesRangeQuery.java | 2 +- .../OpenSearchGenerationException.java | 2 +- .../opensearch/OpenSearchParseException.java | 4 +-- .../OpenSearchSecurityException.java | 4 +-- .../opensearch/OpenSearchServerException.java | 2 +- .../opensearch/OpenSearchStatusException.java | 6 ++-- .../OpenSearchTimeoutException.java | 2 +- .../ResourceAlreadyExistsException.java | 6 ++-- .../opensearch/ResourceNotFoundException.java | 4 +-- .../action/ActionListenerResponseHandler.java | 4 +-- .../org/opensearch/action/ActionRequest.java | 4 +-- .../org/opensearch/action/ActionResponse.java | 2 +- .../org/opensearch/action/ActionType.java | 5 ++-- .../opensearch/action/DocWriteRequest.java | 6 ++-- .../opensearch/action/DocWriteResponse.java | 14 +++++----- .../action/FailedNodeException.java | 4 +-- .../NoShardAvailableActionException.java | 6 ++-- .../action/NoSuchNodeException.java | 2 +- .../opensearch/action/OriginalIndices.java | 4 +-- .../action/PrimaryMissingActionException.java | 2 +- .../opensearch/action/RequestValidators.java | 2 +- .../action/RoutingMissingException.java | 6 ++-- .../action/TaskOperationFailure.java | 8 +++--- .../action/TimestampParsingException.java | 4 +-- .../action/UnavailableShardsException.java | 6 ++-- .../ClusterAllocationExplainRequest.java | 4 +-- .../ClusterAllocationExplainResponse.java | 4 +-- .../ClusterAllocationExplanation.java | 8 +++--- ...ansportClusterAllocationExplainAction.java | 2 +- .../AddVotingConfigExclusionsRequest.java | 4 +-- .../AddVotingConfigExclusionsResponse.java | 4 +-- .../ClearVotingConfigExclusionsRequest.java | 4 +-- .../ClearVotingConfigExclusionsResponse.java | 4 +-- ...nsportAddVotingConfigExclusionsAction.java | 2 +- ...portClearVotingConfigExclusionsAction.java | 2 +- .../DeleteDecommissionStateRequest.java | 4 +-- .../DeleteDecommissionStateResponse.java | 4 +-- ...ransportDeleteDecommissionStateAction.java | 2 +- .../get/GetDecommissionStateRequest.java | 4 +-- .../get/GetDecommissionStateResponse.java | 6 ++-- .../TransportGetDecommissionStateAction.java | 2 +- .../awareness/put/DecommissionRequest.java | 4 +-- .../awareness/put/DecommissionResponse.java | 4 +-- .../put/TransportDecommissionAction.java | 2 +- .../cluster/health/ClusterHealthRequest.java | 4 +-- .../cluster/health/ClusterHealthResponse.java | 6 ++-- .../health/TransportClusterHealthAction.java | 2 +- .../node/hotthreads/NodeHotThreads.java | 4 +-- .../hotthreads/NodesHotThreadsRequest.java | 4 +-- .../hotthreads/NodesHotThreadsResponse.java | 4 +-- .../TransportNodesHotThreadsAction.java | 4 +-- .../admin/cluster/node/info/NodeInfo.java | 4 +-- .../cluster/node/info/NodesInfoRequest.java | 4 +-- .../cluster/node/info/NodesInfoResponse.java | 4 +-- .../cluster/node/info/PluginsAndModules.java | 4 +-- .../node/info/TransportNodesInfoAction.java | 4 +-- .../node/liveness/LivenessRequest.java | 2 +- .../node/liveness/LivenessResponse.java | 4 +-- .../NodesReloadSecureSettingsRequest.java | 10 +++---- ...desReloadSecureSettingsRequestBuilder.java | 2 +- .../NodesReloadSecureSettingsResponse.java | 4 +-- ...nsportNodesReloadSecureSettingsAction.java | 6 ++-- .../admin/cluster/node/stats/NodeStats.java | 4 +-- .../cluster/node/stats/NodesStatsRequest.java | 4 +-- .../node/stats/NodesStatsResponse.java | 4 +-- .../node/stats/TransportNodesStatsAction.java | 4 +-- .../node/tasks/cancel/CancelTasksRequest.java | 4 +-- .../tasks/cancel/CancelTasksResponse.java | 2 +- .../node/tasks/get/GetTaskRequest.java | 4 +-- .../node/tasks/get/GetTaskResponse.java | 4 +-- .../node/tasks/list/ListTasksRequest.java | 4 +-- .../node/tasks/list/ListTasksResponse.java | 4 +-- .../admin/cluster/node/usage/NodeUsage.java | 4 +-- .../cluster/node/usage/NodesUsageRequest.java | 4 +-- .../node/usage/NodesUsageResponse.java | 4 +-- .../node/usage/TransportNodesUsageAction.java | 4 +-- .../cluster/remote/RemoteInfoRequest.java | 2 +- .../cluster/remote/RemoteInfoResponse.java | 4 +-- .../restore/RestoreRemoteStoreRequest.java | 4 +-- .../restore/RestoreRemoteStoreResponse.java | 6 ++-- .../TransportRestoreRemoteStoreAction.java | 2 +- .../remotestore/stats/RemoteStoreStats.java | 6 ++-- .../stats/RemoteStoreStatsRequest.java | 4 +-- .../stats/RemoteStoreStatsResponse.java | 6 ++-- .../TransportRemoteStoreStatsAction.java | 4 +-- .../cleanup/CleanupRepositoryRequest.java | 4 +-- .../cleanup/CleanupRepositoryResponse.java | 4 +-- .../TransportCleanupRepositoryAction.java | 2 +- .../delete/DeleteRepositoryRequest.java | 4 +-- .../TransportDeleteRepositoryAction.java | 2 +- .../get/GetRepositoriesRequest.java | 4 +-- .../get/GetRepositoriesResponse.java | 6 ++-- .../get/TransportGetRepositoriesAction.java | 2 +- .../put/PutRepositoryRequest.java | 4 +-- .../put/TransportPutRepositoryAction.java | 2 +- .../TransportVerifyRepositoryAction.java | 2 +- .../verify/VerifyRepositoryRequest.java | 4 +-- .../verify/VerifyRepositoryResponse.java | 6 ++-- .../reroute/ClusterRerouteRequest.java | 4 +-- .../reroute/ClusterRerouteResponse.java | 4 +-- .../TransportClusterRerouteAction.java | 2 +- .../settings/ClusterGetSettingsResponse.java | 2 +- .../ClusterUpdateSettingsRequest.java | 4 +-- .../ClusterUpdateSettingsResponse.java | 4 +-- .../TransportClusterUpdateSettingsAction.java | 2 +- .../shards/ClusterSearchShardsGroup.java | 8 +++--- .../shards/ClusterSearchShardsRequest.java | 4 +-- .../shards/ClusterSearchShardsResponse.java | 4 +-- .../TransportClusterSearchShardsAction.java | 4 +-- .../ClusterDeleteWeightedRoutingRequest.java | 6 ++-- .../ClusterDeleteWeightedRoutingResponse.java | 4 +-- .../TransportDeleteWeightedRoutingAction.java | 2 +- .../get/ClusterGetWeightedRoutingRequest.java | 4 +-- .../ClusterGetWeightedRoutingResponse.java | 6 ++-- .../TransportGetWeightedRoutingAction.java | 2 +- .../put/ClusterPutWeightedRoutingRequest.java | 6 ++-- .../ClusterPutWeightedRoutingResponse.java | 2 +- .../TransportAddWeightedRoutingAction.java | 2 +- .../snapshots/clone/CloneSnapshotRequest.java | 4 +-- .../clone/TransportCloneSnapshotAction.java | 2 +- .../create/CreateSnapshotRequest.java | 6 ++-- .../create/CreateSnapshotResponse.java | 6 ++-- .../create/TransportCreateSnapshotAction.java | 2 +- .../delete/DeleteSnapshotRequest.java | 4 +-- .../delete/TransportDeleteSnapshotAction.java | 2 +- .../snapshots/get/GetSnapshotsRequest.java | 4 +-- .../snapshots/get/GetSnapshotsResponse.java | 4 +-- .../get/TransportGetSnapshotsAction.java | 2 +- .../restore/RestoreClusterStateListener.java | 2 +- .../restore/RestoreSnapshotRequest.java | 4 +-- .../restore/RestoreSnapshotResponse.java | 6 ++-- .../TransportRestoreSnapshotAction.java | 2 +- .../status/SnapshotIndexShardStatus.java | 10 +++---- .../snapshots/status/SnapshotIndexStatus.java | 2 +- .../snapshots/status/SnapshotStats.java | 8 +++--- .../snapshots/status/SnapshotStatus.java | 6 ++-- .../status/SnapshotsStatusRequest.java | 4 +-- .../status/SnapshotsStatusResponse.java | 4 +-- .../status/TransportNodesSnapshotsStatus.java | 6 ++-- .../TransportSnapshotsStatusAction.java | 4 +-- .../cluster/state/ClusterStateRequest.java | 4 +-- .../cluster/state/ClusterStateResponse.java | 4 +-- .../state/TransportClusterStateAction.java | 2 +- .../admin/cluster/stats/AnalysisStats.java | 6 ++-- .../stats/ClusterStatsNodeResponse.java | 4 +-- .../cluster/stats/ClusterStatsRequest.java | 4 +-- .../cluster/stats/ClusterStatsResponse.java | 4 +-- .../cluster/stats/IndexFeatureStats.java | 6 ++-- .../admin/cluster/stats/MappingStats.java | 6 ++-- .../stats/TransportClusterStatsAction.java | 4 +-- .../DeleteStoredScriptRequest.java | 4 +-- .../GetScriptContextRequest.java | 2 +- .../GetScriptContextResponse.java | 6 ++-- .../GetScriptLanguageRequest.java | 2 +- .../GetScriptLanguageResponse.java | 8 +++--- .../storedscripts/GetStoredScriptRequest.java | 4 +-- .../GetStoredScriptResponse.java | 6 ++-- .../storedscripts/PutStoredScriptRequest.java | 6 ++-- .../PutStoredScriptRequestBuilder.java | 2 +- .../TransportDeleteStoredScriptAction.java | 2 +- .../TransportGetStoredScriptAction.java | 2 +- .../TransportPutStoredScriptAction.java | 2 +- .../tasks/PendingClusterTasksRequest.java | 2 +- .../tasks/PendingClusterTasksResponse.java | 4 +-- .../TransportPendingClusterTasksAction.java | 2 +- .../action/admin/indices/alias/Alias.java | 8 +++--- .../indices/alias/IndicesAliasesRequest.java | 10 +++---- .../alias/TransportIndicesAliasesAction.java | 4 +-- .../indices/alias/get/GetAliasesRequest.java | 4 +-- .../indices/alias/get/GetAliasesResponse.java | 4 +-- .../alias/get/TransportGetAliasesAction.java | 2 +- .../admin/indices/analyze/AnalyzeAction.java | 6 ++-- .../analyze/TransportAnalyzeAction.java | 4 +-- .../cache/clear/ClearIndicesCacheRequest.java | 4 +-- .../clear/ClearIndicesCacheResponse.java | 4 +-- .../TransportClearIndicesCacheAction.java | 4 +-- .../indices/close/CloseIndexRequest.java | 4 +-- .../indices/close/CloseIndexResponse.java | 10 +++---- .../close/TransportCloseIndexAction.java | 4 +-- ...TransportVerifyShardBeforeCloseAction.java | 6 ++-- .../indices/create/AutoCreateAction.java | 2 +- .../CreateIndexClusterStateUpdateRequest.java | 2 +- .../indices/create/CreateIndexRequest.java | 8 +++--- .../create/CreateIndexRequestBuilder.java | 2 +- .../indices/create/CreateIndexResponse.java | 4 +-- .../create/TransportCreateIndexAction.java | 2 +- .../indices/dangling/DanglingIndexInfo.java | 6 ++-- .../delete/DeleteDanglingIndexRequest.java | 4 +-- .../TransportDeleteDanglingIndexAction.java | 4 +-- .../find/FindDanglingIndexRequest.java | 4 +-- .../find/FindDanglingIndexResponse.java | 4 +-- .../find/NodeFindDanglingIndexRequest.java | 4 +-- .../find/NodeFindDanglingIndexResponse.java | 4 +-- .../TransportFindDanglingIndexAction.java | 2 +- .../ImportDanglingIndexRequest.java | 4 +-- .../list/ListDanglingIndicesRequest.java | 4 +-- .../list/ListDanglingIndicesResponse.java | 6 ++-- .../list/NodeListDanglingIndicesRequest.java | 4 +-- .../list/NodeListDanglingIndicesResponse.java | 4 +-- .../TransportListDanglingIndicesAction.java | 2 +- .../datastream/CreateDataStreamAction.java | 4 +-- .../datastream/DataStreamsStatsAction.java | 8 +++--- .../datastream/DeleteDataStreamAction.java | 6 ++-- .../datastream/GetDataStreamAction.java | 6 ++-- .../indices/delete/DeleteIndexRequest.java | 4 +-- .../delete/TransportDeleteIndexAction.java | 4 +-- .../exists/indices/IndicesExistsRequest.java | 4 +-- .../exists/indices/IndicesExistsResponse.java | 4 +-- .../indices/TransportIndicesExistsAction.java | 2 +- .../admin/indices/flush/FlushRequest.java | 4 +-- .../admin/indices/flush/FlushResponse.java | 4 +-- .../indices/flush/ShardFlushRequest.java | 6 ++-- .../indices/flush/TransportFlushAction.java | 4 +-- .../flush/TransportShardFlushAction.java | 6 ++-- .../indices/forcemerge/ForceMergeRequest.java | 4 +-- .../forcemerge/ForceMergeResponse.java | 4 +-- .../forcemerge/TransportForceMergeAction.java | 4 +-- .../admin/indices/get/GetIndexRequest.java | 4 +-- .../admin/indices/get/GetIndexResponse.java | 4 +-- .../indices/get/TransportGetIndexAction.java | 2 +- .../get/GetFieldMappingsIndexRequest.java | 4 +-- .../mapping/get/GetFieldMappingsRequest.java | 4 +-- .../mapping/get/GetFieldMappingsResponse.java | 6 ++-- .../mapping/get/GetMappingsRequest.java | 2 +- .../mapping/get/GetMappingsResponse.java | 4 +-- .../TransportGetFieldMappingsIndexAction.java | 6 ++-- .../get/TransportGetMappingsAction.java | 2 +- .../mapping/put/PutMappingRequest.java | 10 +++---- .../mapping/put/PutMappingRequestBuilder.java | 2 +- .../put/TransportAutoPutMappingAction.java | 4 +-- .../put/TransportPutMappingAction.java | 4 +-- .../admin/indices/open/OpenIndexRequest.java | 4 +-- .../admin/indices/open/OpenIndexResponse.java | 4 +-- .../open/TransportOpenIndexAction.java | 4 +-- .../readonly/AddIndexBlockRequest.java | 4 +-- .../readonly/AddIndexBlockResponse.java | 10 +++---- .../TransportAddIndexBlockAction.java | 4 +-- .../TransportVerifyShardIndexBlockAction.java | 6 ++-- .../indices/recovery/RecoveryRequest.java | 4 +-- .../indices/recovery/RecoveryResponse.java | 6 ++-- .../recovery/TransportRecoveryAction.java | 4 +-- .../admin/indices/refresh/RefreshRequest.java | 2 +- .../indices/refresh/RefreshResponse.java | 4 +-- .../refresh/TransportRefreshAction.java | 4 +-- .../refresh/TransportShardRefreshAction.java | 2 +- .../SegmentReplicationShardStatsResponse.java | 6 ++-- .../SegmentReplicationStatsRequest.java | 4 +-- .../SegmentReplicationStatsResponse.java | 6 ++-- ...ransportSegmentReplicationStatsAction.java | 6 ++-- .../indices/resolve/ResolveIndexAction.java | 6 ++-- .../admin/indices/rollover/Condition.java | 2 +- .../indices/rollover/MaxAgeCondition.java | 4 +-- .../indices/rollover/MaxDocsCondition.java | 4 +-- .../indices/rollover/MaxSizeCondition.java | 4 +-- .../admin/indices/rollover/RolloverInfo.java | 6 ++-- .../indices/rollover/RolloverRequest.java | 4 +-- .../indices/rollover/RolloverResponse.java | 4 +-- .../rollover/TransportRolloverAction.java | 2 +- .../indices/segments/IndexShardSegments.java | 2 +- .../segments/IndicesSegmentResponse.java | 6 ++-- .../segments/IndicesSegmentsRequest.java | 4 +-- .../indices/segments/PitSegmentsRequest.java | 4 +-- .../admin/indices/segments/ShardSegments.java | 6 ++-- .../TransportIndicesSegmentsAction.java | 4 +-- .../segments/TransportPitSegmentsAction.java | 10 +++---- .../settings/get/GetSettingsRequest.java | 4 +-- .../settings/get/GetSettingsResponse.java | 6 ++-- .../get/TransportGetSettingsAction.java | 4 +-- .../put/TransportUpdateSettingsAction.java | 4 +-- .../settings/put/UpdateSettingsRequest.java | 4 +-- .../shards/IndicesShardStoresRequest.java | 4 +-- .../shards/IndicesShardStoresResponse.java | 8 +++--- .../TransportIndicesShardStoresAction.java | 4 +-- .../admin/indices/shrink/ResizeRequest.java | 4 +-- .../admin/indices/shrink/ResizeResponse.java | 2 +- .../indices/shrink/TransportResizeAction.java | 4 +-- .../admin/indices/stats/CommonStats.java | 6 ++-- .../admin/indices/stats/CommonStatsFlags.java | 6 ++-- .../admin/indices/stats/IndexShardStats.java | 8 +++--- .../indices/stats/IndicesStatsRequest.java | 4 +-- .../indices/stats/IndicesStatsResponse.java | 8 +++--- .../admin/indices/stats/ShardStats.java | 6 ++-- .../stats/TransportIndicesStatsAction.java | 4 +-- .../delete/DeleteComponentTemplateAction.java | 4 +-- .../DeleteComposableIndexTemplateAction.java | 4 +-- .../delete/DeleteIndexTemplateRequest.java | 4 +-- ...ransportDeleteComponentTemplateAction.java | 2 +- ...rtDeleteComposableIndexTemplateAction.java | 2 +- .../TransportDeleteIndexTemplateAction.java | 2 +- .../get/GetComponentTemplateAction.java | 4 +-- .../get/GetComposableIndexTemplateAction.java | 4 +-- .../get/GetIndexTemplatesRequest.java | 4 +-- .../get/GetIndexTemplatesResponse.java | 4 +-- .../TransportGetComponentTemplateAction.java | 2 +- ...sportGetComposableIndexTemplateAction.java | 2 +- .../get/TransportGetIndexTemplatesAction.java | 2 +- .../post/SimulateIndexTemplateRequest.java | 4 +-- .../post/SimulateIndexTemplateResponse.java | 4 +-- .../template/post/SimulateTemplateAction.java | 4 +-- .../TransportSimulateIndexTemplateAction.java | 2 +- .../post/TransportSimulateTemplateAction.java | 2 +- .../put/PutComponentTemplateAction.java | 4 +-- .../put/PutComposableIndexTemplateAction.java | 4 +-- .../template/put/PutIndexTemplateRequest.java | 8 +++--- .../put/PutIndexTemplateRequestBuilder.java | 2 +- .../TransportPutComponentTemplateAction.java | 2 +- ...sportPutComposableIndexTemplateAction.java | 2 +- .../put/TransportPutIndexTemplateAction.java | 2 +- .../upgrade/get/IndexShardUpgradeStatus.java | 2 +- .../upgrade/get/ShardUpgradeStatus.java | 4 +-- .../get/TransportUpgradeStatusAction.java | 4 +-- .../upgrade/get/UpgradeStatusRequest.java | 2 +- .../upgrade/get/UpgradeStatusResponse.java | 6 ++-- .../upgrade/post/ShardUpgradeResult.java | 8 +++--- .../upgrade/post/TransportUpgradeAction.java | 4 +-- .../post/TransportUpgradeSettingsAction.java | 2 +- .../indices/upgrade/post/UpgradeRequest.java | 4 +-- .../indices/upgrade/post/UpgradeResponse.java | 6 ++-- .../upgrade/post/UpgradeSettingsRequest.java | 4 +-- .../validate/query/QueryExplanation.java | 6 ++-- .../query/ShardValidateQueryRequest.java | 6 ++-- .../query/ShardValidateQueryResponse.java | 6 ++-- .../query/TransportValidateQueryAction.java | 6 ++-- .../validate/query/ValidateQueryRequest.java | 4 +-- .../validate/query/ValidateQueryResponse.java | 6 ++-- .../action/bulk/BulkItemRequest.java | 8 +++--- .../action/bulk/BulkItemResponse.java | 14 +++++----- .../opensearch/action/bulk/BulkProcessor.java | 2 +- .../opensearch/action/bulk/BulkRequest.java | 8 +++--- .../action/bulk/BulkRequestParser.java | 4 +-- .../opensearch/action/bulk/BulkResponse.java | 12 ++++---- .../action/bulk/BulkShardRequest.java | 6 ++-- .../action/bulk/BulkShardResponse.java | 6 ++-- .../action/bulk/MappingUpdatePerformer.java | 2 +- .../org/opensearch/action/bulk/Retry.java | 2 +- .../action/bulk/TransportBulkAction.java | 4 +-- .../action/bulk/TransportShardBulkAction.java | 8 +++--- .../TransportSingleItemBulkWriteAction.java | 2 +- .../action/delete/DeleteRequest.java | 6 ++-- .../action/delete/DeleteResponse.java | 8 +++--- .../action/explain/ExplainRequest.java | 4 +-- .../action/explain/ExplainResponse.java | 6 ++-- .../explain/TransportExplainAction.java | 4 +-- .../action/fieldcaps/FieldCapabilities.java | 6 ++-- .../FieldCapabilitiesIndexRequest.java | 6 ++-- .../FieldCapabilitiesIndexResponse.java | 6 ++-- .../fieldcaps/FieldCapabilitiesRequest.java | 4 +-- .../fieldcaps/FieldCapabilitiesResponse.java | 6 ++-- .../fieldcaps/IndexFieldCapabilities.java | 6 ++-- ...TransportFieldCapabilitiesIndexAction.java | 4 +-- .../org/opensearch/action/get/GetRequest.java | 4 +-- .../opensearch/action/get/GetResponse.java | 8 +++--- .../action/get/MultiGetItemResponse.java | 6 ++-- .../action/get/MultiGetRequest.java | 8 +++--- .../action/get/MultiGetResponse.java | 6 ++-- .../action/get/MultiGetShardRequest.java | 4 +-- .../action/get/MultiGetShardResponse.java | 4 +-- .../action/get/TransportGetAction.java | 4 +-- .../action/get/TransportMultiGetAction.java | 2 +- .../get/TransportShardMultiGetAction.java | 4 +-- .../opensearch/action/index/IndexRequest.java | 10 +++---- .../action/index/IndexRequestBuilder.java | 2 +- .../action/index/IndexResponse.java | 8 +++--- .../action/ingest/DeletePipelineRequest.java | 4 +-- .../ingest/DeletePipelineTransportAction.java | 2 +- .../action/ingest/GetPipelineRequest.java | 4 +-- .../action/ingest/GetPipelineResponse.java | 10 +++---- .../ingest/GetPipelineTransportAction.java | 2 +- .../action/ingest/PutPipelineRequest.java | 6 ++-- .../ingest/PutPipelineRequestBuilder.java | 2 +- .../ingest/PutPipelineTransportAction.java | 2 +- .../ingest/SimulateDocumentBaseResult.java | 4 +-- .../action/ingest/SimulateDocumentResult.java | 2 +- .../ingest/SimulateDocumentVerboseResult.java | 4 +-- .../ingest/SimulatePipelineRequest.java | 6 ++-- .../SimulatePipelineRequestBuilder.java | 2 +- .../ingest/SimulatePipelineResponse.java | 6 ++-- .../SimulatePipelineTransportAction.java | 2 +- .../ingest/SimulateProcessorResult.java | 6 ++-- .../ingest/WriteableIngestDocument.java | 6 ++-- .../opensearch/action/main/MainRequest.java | 2 +- .../opensearch/action/main/MainResponse.java | 4 +-- .../resync/ResyncReplicationRequest.java | 6 ++-- .../resync/ResyncReplicationResponse.java | 2 +- .../TransportResyncReplicationAction.java | 2 +- .../search/AbstractSearchAsyncAction.java | 4 +-- .../action/search/ClearScrollRequest.java | 4 +-- .../action/search/ClearScrollResponse.java | 10 +++---- .../action/search/CreatePitController.java | 4 +-- .../action/search/CreatePitRequest.java | 4 +-- .../action/search/CreatePitResponse.java | 8 +++--- .../action/search/DeletePitInfo.java | 6 ++-- .../action/search/DeletePitRequest.java | 4 +-- .../action/search/DeletePitResponse.java | 8 +++--- .../search/DeleteSearchPipelineRequest.java | 4 +-- .../DeleteSearchPipelineTransportAction.java | 2 +- .../action/search/GetAllPitNodeRequest.java | 4 +-- .../action/search/GetAllPitNodeResponse.java | 4 +-- .../action/search/GetAllPitNodesRequest.java | 4 +-- .../action/search/GetAllPitNodesResponse.java | 4 +-- .../search/GetSearchPipelineRequest.java | 4 +-- .../search/GetSearchPipelineResponse.java | 10 +++---- .../GetSearchPipelineTransportAction.java | 2 +- .../opensearch/action/search/ListPitInfo.java | 6 ++-- .../action/search/MultiSearchRequest.java | 6 ++-- .../action/search/MultiSearchResponse.java | 6 ++-- .../search/PitSearchContextIdForNode.java | 6 ++-- .../opensearch/action/search/PitService.java | 2 +- .../search/PutSearchPipelineRequest.java | 6 ++-- .../PutSearchPipelineTransportAction.java | 2 +- .../search/QueryPhaseResultConsumer.java | 2 +- .../search/ReduceSearchPhaseException.java | 2 +- .../action/search/SearchContextId.java | 14 +++++----- .../action/search/SearchContextIdForNode.java | 6 ++-- .../action/search/SearchPhaseController.java | 2 +- .../search/SearchPhaseExecutionException.java | 8 +++--- .../action/search/SearchRequest.java | 4 +-- .../action/search/SearchResponse.java | 10 +++---- .../action/search/SearchResponseMerger.java | 2 +- .../action/search/SearchResponseSections.java | 2 +- .../action/search/SearchScrollRequest.java | 4 +-- .../opensearch/action/search/SearchShard.java | 2 +- .../action/search/SearchShardIterator.java | 2 +- .../action/search/SearchTransportService.java | 6 ++-- .../action/search/ShardSearchFailure.java | 14 +++++----- .../search/TransportClearScrollAction.java | 2 +- .../search/TransportCreatePitAction.java | 8 +++--- .../search/TransportDeletePitAction.java | 2 +- .../search/TransportGetAllPitsAction.java | 2 +- .../search/TransportMultiSearchAction.java | 2 +- .../action/search/TransportSearchAction.java | 8 +++--- .../action/search/TransportSearchHelper.java | 4 +-- .../search/TransportSearchScrollAction.java | 2 +- .../search/UpdatePitContextRequest.java | 4 +-- .../search/UpdatePitContextResponse.java | 4 +-- .../action/support/ActiveShardCount.java | 6 ++-- .../support/HandledTransportAction.java | 2 +- .../action/support/IndicesOptions.java | 4 +-- .../action/support/WriteRequest.java | 6 ++-- .../support/broadcast/BroadcastRequest.java | 4 +-- .../support/broadcast/BroadcastResponse.java | 10 +++---- ...roadcastShardOperationFailedException.java | 4 +-- .../broadcast/BroadcastShardRequest.java | 6 ++-- .../broadcast/BroadcastShardResponse.java | 6 ++-- .../broadcast/TransportBroadcastAction.java | 4 +-- .../node/TransportBroadcastByNodeAction.java | 8 +++--- .../ClusterManagerNodeReadRequest.java | 4 +-- .../ClusterManagerNodeRequest.java | 4 +-- .../TransportClusterManagerNodeAction.java | 4 +-- ...TransportClusterManagerNodeReadAction.java | 2 +- .../info/ClusterInfoRequest.java | 4 +-- .../info/TransportClusterInfoAction.java | 2 +- .../support/master/AcknowledgedRequest.java | 4 +-- .../support/master/AcknowledgedResponse.java | 4 +-- .../support/master/MasterNodeReadRequest.java | 2 +- .../support/master/MasterNodeRequest.java | 2 +- .../master/ShardsAcknowledgedResponse.java | 4 +-- .../master/TransportMasterNodeAction.java | 2 +- .../master/TransportMasterNodeReadAction.java | 2 +- .../master/info/ClusterInfoRequest.java | 2 +- .../info/TransportClusterInfoAction.java | 2 +- .../support/nodes/BaseNodeResponse.java | 4 +-- .../support/nodes/BaseNodesRequest.java | 4 +-- .../support/nodes/BaseNodesResponse.java | 4 +-- .../support/nodes/TransportNodesAction.java | 4 +-- .../replication/BasicReplicationRequest.java | 4 +-- .../PendingReplicationActions.java | 2 +- .../replication/ReplicatedWriteRequest.java | 6 ++-- .../replication/ReplicationOperation.java | 6 ++-- .../replication/ReplicationRequest.java | 6 ++-- .../replication/ReplicationResponse.java | 14 +++++----- .../support/replication/ReplicationTask.java | 4 +-- .../TransportBroadcastReplicationAction.java | 6 ++-- .../TransportReplicationAction.java | 8 +++--- .../replication/TransportWriteAction.java | 4 +-- .../InstanceShardOperationRequest.java | 6 ++-- ...ransportInstanceSingleOperationAction.java | 6 ++-- .../single/shard/SingleShardRequest.java | 6 ++-- .../shard/TransportSingleShardAction.java | 6 ++-- .../support/tasks/BaseTasksRequest.java | 4 +-- .../support/tasks/BaseTasksResponse.java | 4 +-- .../support/tasks/TransportTasksAction.java | 6 ++-- .../MultiTermVectorsItemResponse.java | 6 ++-- .../termvectors/MultiTermVectorsRequest.java | 4 +-- .../termvectors/MultiTermVectorsResponse.java | 6 ++-- .../MultiTermVectorsShardRequest.java | 4 +-- .../MultiTermVectorsShardResponse.java | 4 +-- .../action/termvectors/TermVectorsFields.java | 4 +-- .../termvectors/TermVectorsRequest.java | 8 +++--- .../termvectors/TermVectorsResponse.java | 8 +++--- .../action/termvectors/TermVectorsWriter.java | 2 +- .../TransportMultiTermVectorsAction.java | 2 +- .../TransportShardMultiTermsVectorAction.java | 4 +-- .../TransportTermVectorsAction.java | 4 +-- .../action/update/TransportUpdateAction.java | 8 +++--- .../action/update/UpdateHelper.java | 6 ++-- .../action/update/UpdateRequest.java | 6 ++-- .../action/update/UpdateResponse.java | 10 +++---- .../org/opensearch/bootstrap/Bootstrap.java | 2 +- .../opensearch/cli/KeyStoreAwareCommand.java | 2 +- .../opensearch/client/ClusterAdminClient.java | 2 +- .../opensearch/client/node/NodeClient.java | 2 +- .../client/support/AbstractClient.java | 2 +- .../transport/NoNodeAvailableException.java | 4 +-- .../opensearch/cluster/AbstractDiffable.java | 4 +-- .../cluster/AbstractNamedDiffable.java | 6 ++-- .../cluster/ClusterChangedEvent.java | 2 +- .../org/opensearch/cluster/ClusterInfo.java | 8 +++--- .../org/opensearch/cluster/ClusterModule.java | 6 ++-- .../org/opensearch/cluster/ClusterName.java | 6 ++-- .../cluster/ClusterSettingsResponse.java | 4 +-- .../org/opensearch/cluster/ClusterState.java | 12 ++++---- .../java/org/opensearch/cluster/Diff.java | 2 +- .../java/org/opensearch/cluster/Diffable.java | 2 +- .../org/opensearch/cluster/DiffableUtils.java | 6 ++-- .../org/opensearch/cluster/DiskUsage.java | 6 ++-- ...ompatibleClusterStateVersionException.java | 2 +- .../org/opensearch/cluster/NamedDiff.java | 2 +- .../org/opensearch/cluster/NamedDiffable.java | 2 +- .../cluster/NamedDiffableValueSerializer.java | 2 +- .../cluster/NotClusterManagerException.java | 2 +- .../cluster/NotMasterException.java | 2 +- .../cluster/RepositoryCleanupInProgress.java | 6 ++-- .../opensearch/cluster/RestoreInProgress.java | 8 +++--- .../cluster/SnapshotDeletionsInProgress.java | 6 ++-- .../cluster/SnapshotsInProgress.java | 8 +++--- .../ack/IndicesClusterStateUpdateRequest.java | 2 +- .../action/index/MappingUpdatedAction.java | 2 +- .../index/NodeMappingRefreshAction.java | 4 +-- .../action/shard/ShardStateAction.java | 6 ++-- .../ClusterAwarenessAttributeValueHealth.java | 8 +++--- .../ClusterAwarenessAttributesHealth.java | 8 +++--- .../ClusterAwarenessHealth.java | 6 ++-- .../cluster/block/ClusterBlock.java | 8 +++--- .../cluster/block/ClusterBlockException.java | 6 ++-- .../cluster/block/ClusterBlocks.java | 6 ++-- .../block/IndexCreateBlockException.java | 2 +- .../coordination/ApplyCommitRequest.java | 4 +-- .../coordination/CompressedStreamUtils.java | 14 +++++----- .../coordination/CoordinationMetadata.java | 6 ++-- .../CoordinationStateRejectedException.java | 2 +- .../cluster/coordination/Coordinator.java | 2 +- .../FailedToCommitClusterStateException.java | 2 +- .../coordination/FollowersChecker.java | 4 +-- .../opensearch/cluster/coordination/Join.java | 6 ++-- .../cluster/coordination/JoinHelper.java | 6 ++-- .../cluster/coordination/JoinRequest.java | 4 +-- .../cluster/coordination/LeaderChecker.java | 4 +-- .../NoClusterManagerBlockService.java | 2 +- .../NodeHealthCheckFailureException.java | 2 +- .../coordination/OpenSearchNodeCommand.java | 2 +- .../cluster/coordination/PeersResponse.java | 4 +-- .../PendingClusterStateStats.java | 6 ++-- .../coordination/PreVoteCollector.java | 2 +- .../cluster/coordination/PreVoteRequest.java | 4 +-- .../cluster/coordination/PreVoteResponse.java | 4 +-- .../PublicationTransportHandler.java | 6 ++-- .../PublishClusterStateStats.java | 6 ++-- .../cluster/coordination/PublishResponse.java | 6 ++-- .../coordination/PublishWithJoinResponse.java | 4 +-- .../coordination/StartJoinRequest.java | 4 +-- .../coordination/TermVersionRequest.java | 6 ++-- .../coordination/ValidateJoinRequest.java | 4 +-- .../decommission/DecommissionAttribute.java | 6 ++-- .../DecommissionAttributeMetadata.java | 4 +-- .../decommission/DecommissionController.java | 2 +- .../DecommissioningFailedException.java | 6 ++-- .../NodeDecommissionedException.java | 4 +-- .../cluster/health/ClusterHealthStatus.java | 6 ++-- .../cluster/health/ClusterIndexHealth.java | 8 +++--- .../cluster/health/ClusterShardHealth.java | 8 +++--- .../cluster/health/ClusterStateHealth.java | 8 +++--- .../cluster/metadata/AliasMetadata.java | 6 ++-- .../cluster/metadata/AliasValidator.java | 2 +- .../cluster/metadata/ComponentTemplate.java | 4 +-- .../metadata/ComponentTemplateMetadata.java | 4 +-- .../metadata/ComposableIndexTemplate.java | 6 ++-- .../ComposableIndexTemplateMetadata.java | 4 +-- .../cluster/metadata/DataStream.java | 8 +++--- .../cluster/metadata/DataStreamMetadata.java | 4 +-- .../cluster/metadata/DiffableStringMap.java | 4 +-- .../cluster/metadata/IndexGraveyard.java | 8 +++--- .../cluster/metadata/IndexMetadata.java | 12 ++++---- .../metadata/IndexNameExpressionResolver.java | 2 +- .../metadata/IndexTemplateMetadata.java | 4 +-- .../opensearch/cluster/metadata/Manifest.java | 2 +- .../cluster/metadata/MappingMetadata.java | 6 ++-- .../opensearch/cluster/metadata/Metadata.java | 8 +++--- .../MetadataCreateDataStreamService.java | 2 +- .../metadata/MetadataCreateIndexService.java | 2 +- .../metadata/MetadataDeleteIndexService.java | 2 +- .../metadata/MetadataIndexAliasesService.java | 2 +- .../metadata/MetadataIndexStateService.java | 6 ++-- .../MetadataIndexTemplateService.java | 4 +-- .../metadata/MetadataMappingService.java | 2 +- .../MetadataUpdateSettingsService.java | 2 +- .../ProcessClusterEventTimeoutException.java | 4 +-- .../metadata/RepositoriesMetadata.java | 4 +-- .../cluster/metadata/RepositoryMetadata.java | 6 ++-- .../opensearch/cluster/metadata/Template.java | 4 +-- .../metadata/TemplateUpgradeService.java | 2 +- .../metadata/WeightedRoutingMetadata.java | 4 +-- .../cluster/node/DiscoveryNode.java | 6 ++-- .../cluster/node/DiscoveryNodes.java | 4 +-- .../cluster/routing/AllocationId.java | 6 ++-- .../routing/FailAwareWeightedRouting.java | 4 +-- .../IllegalShardRoutingStateException.java | 4 +-- .../cluster/routing/IndexRoutingTable.java | 8 +++--- .../routing/IndexShardRoutingTable.java | 8 +++--- .../routing/NodeWeighedAwayException.java | 4 +-- .../cluster/routing/OperationRouting.java | 2 +- .../cluster/routing/PlainShardIterator.java | 2 +- ...ferenceBasedSearchNotAllowedException.java | 4 +-- .../cluster/routing/RecoverySource.java | 6 ++-- .../cluster/routing/RoutingException.java | 2 +- .../cluster/routing/RoutingNode.java | 4 +-- .../cluster/routing/RoutingNodes.java | 4 +-- .../cluster/routing/RoutingTable.java | 8 +++--- .../cluster/routing/ShardIterator.java | 2 +- .../cluster/routing/ShardRouting.java | 10 +++---- .../cluster/routing/UnassignedInfo.java | 6 ++-- ...upportedWeightedRoutingStateException.java | 4 +-- .../cluster/routing/WeightedRouting.java | 6 ++-- .../cluster/routing/WeightedRoutingStats.java | 6 ++-- .../AbstractAllocationDecision.java | 6 ++-- .../AllocateUnassignedDecision.java | 4 +-- .../allocation/AllocationDecision.java | 6 ++-- .../allocation/IndexMetadataUpdater.java | 4 +-- .../routing/allocation/MoveDecision.java | 4 +-- .../allocation/NodeAllocationResult.java | 6 ++-- .../allocation/RerouteExplanation.java | 4 +-- .../routing/allocation/RoutingAllocation.java | 2 +- .../allocation/RoutingExplanations.java | 4 +-- .../allocation/ShardAllocationDecision.java | 6 ++-- .../routing/allocation/StaleShard.java | 2 +- .../AbstractAllocateAllocationCommand.java | 4 +-- ...AllocateEmptyPrimaryAllocationCommand.java | 4 +-- .../AllocateReplicaAllocationCommand.java | 2 +- ...AllocateStalePrimaryAllocationCommand.java | 2 +- .../allocation/command/AllocationCommand.java | 2 +- .../command/AllocationCommands.java | 4 +-- .../command/BasePrimaryAllocationCommand.java | 4 +-- .../command/CancelAllocationCommand.java | 6 ++-- .../command/MoveAllocationCommand.java | 4 +-- .../routing/allocation/decider/Decision.java | 6 ++-- .../decider/DiskThresholdDecider.java | 4 +-- .../decider/ResizeAllocationDecider.java | 4 +-- .../ClusterManagerThrottlingException.java | 2 +- .../ClusterManagerThrottlingStats.java | 6 ++-- .../cluster/service/MasterService.java | 2 +- .../cluster/service/PendingClusterTask.java | 8 +++--- .../opensearch/common/FieldMemoryStats.java | 6 ++-- .../java/org/opensearch/common/Priority.java | 4 +-- .../java/org/opensearch/common/Rounding.java | 6 ++-- .../java/org/opensearch/common/Strings.java | 2 +- .../common/blobstore/BlobStoreException.java | 2 +- .../breaker/CircuitBreakingException.java | 6 ++-- .../bytes/RecyclingBytesStreamOutput.java | 9 ++++-- .../bytes/ReleasableBytesReference.java | 3 +- .../common/compress/CompressedXContent.java | 8 +++--- .../common/compress/Compressor.java | 2 +- .../common/compress/CompressorFactory.java | 3 +- .../common/compress/DeflateCompressor.java | 2 +- .../common/compress/NoneCompressor.java | 2 +- .../common/compress/ZstdCompressor.java | 2 +- .../common/document/DocumentField.java | 10 +++---- .../opensearch/common/geo/GeoBoundingBox.java | 6 ++-- .../opensearch/common/geo/GeoDistance.java | 6 ++-- .../org/opensearch/common/geo/GeoPoint.java | 4 +-- .../opensearch/common/geo/GeoShapeType.java | 2 +- .../org/opensearch/common/geo/GeometryIO.java | 4 +-- .../opensearch/common/geo/ShapeRelation.java | 6 ++-- .../common/geo/SpatialStrategy.java | 6 ++-- .../common/geo/builders/CircleBuilder.java | 4 +-- .../common/geo/builders/EnvelopeBuilder.java | 4 +-- .../builders/GeometryCollectionBuilder.java | 4 +-- .../geo/builders/LineStringBuilder.java | 2 +- .../geo/builders/MultiLineStringBuilder.java | 4 +-- .../geo/builders/MultiPointBuilder.java | 2 +- .../geo/builders/MultiPolygonBuilder.java | 4 +-- .../common/geo/builders/PointBuilder.java | 2 +- .../common/geo/builders/PolygonBuilder.java | 4 +-- .../common/geo/builders/ShapeBuilder.java | 6 ++-- .../org/opensearch/common/io/Streams.java | 6 ++-- .../common/io/stream/BytesStreamOutput.java | 12 ++++---- .../common/io/stream/DelayableWriteable.java | 7 ++++- .../common/io/stream/Streamables.java | 4 ++- .../java/org/opensearch/common/joda/Joda.java | 4 +-- .../opensearch/common/logging/Loggers.java | 4 +-- .../org/opensearch/common/lucene/Lucene.java | 4 +-- .../common/lucene/ShardCoreKeyMap.java | 2 +- .../index/OpenSearchDirectoryReader.java | 2 +- .../lucene/index/OpenSearchLeafReader.java | 2 +- .../search/function/CombineFunction.java | 6 ++-- .../function/FieldValueFactorFunction.java | 6 ++-- .../search/function/FunctionScoreQuery.java | 6 ++-- .../common/metrics/OperationStats.java | 6 ++-- .../common/network/NetworkModule.java | 4 +-- .../opensearch/common/rounding/Rounding.java | 6 ++-- .../common/settings/BaseKeyStoreCommand.java | 1 + .../ChangeKeyStorePasswordCommand.java | 1 + .../common/settings/KeyStoreWrapper.java | 1 + .../settings/NoClassSettingsException.java | 2 +- .../common/settings/SecureSetting.java | 1 + .../common/settings/SecureSettings.java | 2 ++ .../opensearch/common/settings/Setting.java | 6 ++-- .../opensearch/common/settings/Settings.java | 7 +++-- .../common/settings/SettingsException.java | 4 +-- .../common/settings/WriteableSetting.java | 6 ++-- .../transport/BoundTransportAddress.java | 6 ++-- .../common/transport/TransportAddress.java | 6 ++-- .../opensearch/common/unit/ByteSizeUnit.java | 6 ++-- .../opensearch/common/unit/ByteSizeValue.java | 6 ++-- .../opensearch/common/unit/DistanceUnit.java | 6 ++-- .../org/opensearch/common/unit/Fuzziness.java | 6 ++-- .../org/opensearch/common/unit/SizeValue.java | 6 ++-- .../opensearch/common/util/AbstractArray.java | 1 + .../org/opensearch/common/util/BigArrays.java | 2 ++ .../opensearch/common/util/BigByteArray.java | 1 + .../opensearch/common/util/BytesRefHash.java | 1 + .../common/util/CancellableThreads.java | 2 +- .../opensearch/common/util/CuckooFilter.java | 6 ++-- .../opensearch/common/util/DoubleArray.java | 2 ++ .../opensearch/common/util/FloatArray.java | 2 ++ .../org/opensearch/common/util/IntArray.java | 2 ++ .../org/opensearch/common/util/LongArray.java | 2 ++ .../opensearch/common/util/ObjectArray.java | 2 ++ .../common/util/PageCacheRecycler.java | 2 +- .../util/SetBackedScalingCuckooFilter.java | 6 ++-- .../common/util/concurrent/ThreadContext.java | 6 ++-- .../UncategorizedExecutionException.java | 2 +- .../xcontent/JsonToStringXContentParser.java | 2 +- .../common/xcontent/ObjectParserHelper.java | 2 +- .../common/xcontent/ParseFieldRegistry.java | 2 +- .../xcontent/StatusToXContentObject.java | 2 +- .../common/xcontent/XContentHelper.java | 4 +-- .../xcontent/XContentOpenSearchExtension.java | 2 +- .../ClusterManagerNotDiscoveredException.java | 4 +-- .../opensearch/discovery/DiscoveryModule.java | 2 +- .../opensearch/discovery/DiscoveryStats.java | 6 ++-- .../discovery/InitializeExtensionRequest.java | 4 +-- .../InitializeExtensionResponse.java | 4 +-- .../MasterNotDiscoveredException.java | 2 +- .../org/opensearch/discovery/PeerFinder.java | 2 +- .../opensearch/discovery/PeersRequest.java | 4 +-- .../env/EnvironmentSettingsResponse.java | 4 +-- .../org/opensearch/env/NodeEnvironment.java | 4 +-- .../java/org/opensearch/env/ShardLock.java | 2 +- .../env/ShardLockObtainFailedException.java | 4 +-- .../extensions/AcknowledgedResponse.java | 4 +-- .../AddSettingsUpdateConsumerRequest.java | 4 +-- .../extensions/DiscoveryExtensionNode.java | 6 ++-- .../extensions/ExtensionDependency.java | 6 ++-- .../ExtensionDependencyResponse.java | 4 +-- .../extensions/ExtensionRequest.java | 4 +-- .../extensions/ExtensionsManager.java | 4 +-- .../extensions/OpenSearchRequest.java | 4 +-- .../extensions/UpdateSettingsRequest.java | 4 +-- .../UpdateSettingsResponseHandler.java | 2 +- .../action/ExtensionActionRequest.java | 4 +-- .../action/ExtensionActionResponse.java | 4 +-- .../action/ExtensionActionUtil.java | 6 ++-- .../ExtensionHandleTransportRequest.java | 4 +-- .../ExtensionTransportActionsHandler.java | 2 +- .../RegisterTransportActionsRequest.java | 4 +-- .../action/RemoteExtensionActionResponse.java | 4 +-- .../TransportActionRequestFromExtension.java | 4 +-- .../extensions/rest/ExtensionRestRequest.java | 6 ++-- .../rest/ExtensionRestResponse.java | 4 +-- .../rest/RegisterRestActionsRequest.java | 4 +-- .../rest/RestExecuteOnExtensionResponse.java | 6 ++-- .../rest/RestInitializeExtensionAction.java | 2 +- .../rest/RestSendToExtensionAction.java | 6 ++-- .../RegisterCustomSettingsRequest.java | 4 +-- .../opensearch/gateway/AsyncShardFetch.java | 2 +- .../gateway/DanglingIndicesState.java | 2 +- .../java/org/opensearch/gateway/Gateway.java | 2 +- .../opensearch/gateway/GatewayAllocator.java | 2 +- .../opensearch/gateway/GatewayException.java | 2 +- .../opensearch/gateway/GatewayService.java | 2 +- .../IncrementalClusterStateWriter.java | 2 +- .../gateway/LocalAllocateDangledIndices.java | 4 +-- .../opensearch/gateway/MetaStateService.java | 2 +- .../gateway/PersistedClusterStateService.java | 6 ++-- .../gateway/PriorityComparator.java | 2 +- .../TransportNodesListGatewayMetaState.java | 4 +-- ...ransportNodesListGatewayStartedShards.java | 6 ++-- .../opensearch/http/BindHttpException.java | 2 +- .../java/org/opensearch/http/CorsHandler.java | 4 +-- .../opensearch/http/DefaultRestChannel.java | 6 ++-- .../org/opensearch/http/HttpException.java | 2 +- .../java/org/opensearch/http/HttpInfo.java | 4 +-- .../opensearch/http/HttpPipelinedRequest.java | 4 +-- .../java/org/opensearch/http/HttpRequest.java | 4 +-- .../java/org/opensearch/http/HttpStats.java | 6 ++-- .../index/AbstractIndexComponent.java | 1 + .../index/CompositeIndexEventListener.java | 3 +- .../org/opensearch/index/IndexComponent.java | 2 ++ .../org/opensearch/index/IndexModule.java | 3 +- .../index/IndexNotFoundException.java | 3 +- .../org/opensearch/index/IndexService.java | 4 +-- .../org/opensearch/index/IndexSettings.java | 1 + .../index/IndexingPressureService.java | 2 +- .../org/opensearch/index/IndexingSlowLog.java | 3 +- .../index/IndicesModuleRequest.java | 5 ++-- .../index/IndicesModuleResponse.java | 4 +-- .../SegmentReplicationPerGroupStats.java | 8 +++--- .../SegmentReplicationPressureService.java | 2 +- .../index/SegmentReplicationShardStats.java | 6 ++-- .../index/SegmentReplicationStats.java | 8 +++--- .../index/SegmentReplicationStatsTracker.java | 2 +- .../index/ShardIndexingPressure.java | 2 +- .../ShardIndexingPressureMemoryManager.java | 2 +- .../index/ShardIndexingPressureStore.java | 2 +- .../index/ShardIndexingPressureTracker.java | 2 +- .../org/opensearch/index/VersionType.java | 6 ++-- .../index/analysis/NameOrDefinition.java | 6 ++-- .../index/cache/bitset/BitsetFilterCache.java | 2 +- .../cache/bitset/ShardBitsetFilterCache.java | 2 +- .../index/cache/query/QueryCacheStats.java | 6 ++-- .../cache/request/RequestCacheStats.java | 6 ++-- .../cache/request/ShardRequestCache.java | 2 +- .../opensearch/index/engine/CommitStats.java | 6 ++-- .../engine/DocumentMissingException.java | 6 ++-- .../DocumentSourceMissingException.java | 6 ++-- .../org/opensearch/index/engine/Engine.java | 4 +-- .../opensearch/index/engine/EngineConfig.java | 2 +- .../index/engine/EngineConfigFactory.java | 2 +- .../EngineCreationFailureException.java | 4 +-- .../index/engine/EngineException.java | 4 +-- .../engine/FlushFailedEngineException.java | 4 +-- .../index/engine/InternalEngine.java | 2 +- .../index/engine/LuceneChangesSnapshot.java | 2 +- .../OpenSearchConcurrentMergeScheduler.java | 2 +- .../index/engine/RecoveryEngineException.java | 6 ++-- .../engine/RefreshFailedEngineException.java | 4 +-- .../org/opensearch/index/engine/Segment.java | 6 ++-- .../index/engine/SegmentsStats.java | 6 ++-- .../engine/SnapshotFailedEngineException.java | 2 +- .../VersionConflictEngineException.java | 6 ++-- .../index/fielddata/FieldDataStats.java | 6 ++-- .../index/fielddata/IndexFieldDataCache.java | 2 +- .../fielddata/IndexFieldDataService.java | 2 +- .../index/fielddata/ShardFieldData.java | 2 +- .../plain/AbstractBinaryDVLeafFieldData.java | 2 +- .../index/fieldvisitor/FieldsVisitor.java | 4 +-- .../opensearch/index/flush/FlushStats.java | 6 ++-- .../org/opensearch/index/get/GetResult.java | 10 +++---- .../org/opensearch/index/get/GetStats.java | 6 ++-- .../opensearch/index/get/ShardGetService.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 4 +-- .../index/mapper/BinaryRangeUtil.java | 2 +- .../index/mapper/CompletionFieldMapper.java | 2 +- .../index/mapper/DocCountFieldMapper.java | 2 +- .../index/mapper/DocumentMapper.java | 4 +-- .../index/mapper/MapperException.java | 2 +- .../index/mapper/MapperParsingException.java | 4 +-- .../index/mapper/ParsedDocument.java | 2 +- .../index/mapper/SourceFieldMapper.java | 2 +- .../index/mapper/SourceToParse.java | 4 +-- .../mapper/StrictDynamicMappingException.java | 4 +-- .../opensearch/index/merge/MergeStats.java | 6 ++-- .../query/AbstractGeometryQueryBuilder.java | 6 ++-- .../index/query/AbstractQueryBuilder.java | 6 ++-- .../index/query/BaseTermQueryBuilder.java | 4 +-- .../index/query/BoolQueryBuilder.java | 6 ++-- .../index/query/BoostingQueryBuilder.java | 6 ++-- .../index/query/CommonTermsQueryBuilder.java | 6 ++-- .../query/ConstantScoreQueryBuilder.java | 6 ++-- .../index/query/DisMaxQueryBuilder.java | 6 ++-- .../query/DistanceFeatureQueryBuilder.java | 6 ++-- .../index/query/ExistsQueryBuilder.java | 6 ++-- .../query/FieldMaskingSpanQueryBuilder.java | 6 ++-- .../index/query/FuzzyQueryBuilder.java | 6 ++-- .../query/GeoBoundingBoxQueryBuilder.java | 6 ++-- .../index/query/GeoDistanceQueryBuilder.java | 6 ++-- .../opensearch/index/query/GeoExecType.java | 6 ++-- .../index/query/GeoPolygonQueryBuilder.java | 6 ++-- .../index/query/GeoShapeQueryBuilder.java | 6 ++-- .../index/query/GeoValidationMethod.java | 6 ++-- .../index/query/IdsQueryBuilder.java | 6 ++-- .../index/query/InnerHitBuilder.java | 8 +++--- .../opensearch/index/query/IntervalMode.java | 6 ++-- .../index/query/IntervalQueryBuilder.java | 6 ++-- .../index/query/IntervalsSourceProvider.java | 10 +++---- .../index/query/MatchAllQueryBuilder.java | 6 ++-- .../query/MatchBoolPrefixQueryBuilder.java | 6 ++-- .../index/query/MatchNoneQueryBuilder.java | 6 ++-- .../query/MatchPhrasePrefixQueryBuilder.java | 6 ++-- .../index/query/MatchPhraseQueryBuilder.java | 6 ++-- .../index/query/MatchQueryBuilder.java | 6 ++-- .../index/query/MoreLikeThisQueryBuilder.java | 10 +++---- .../index/query/MultiMatchQueryBuilder.java | 8 +++--- .../index/query/NestedQueryBuilder.java | 6 ++-- .../org/opensearch/index/query/Operator.java | 6 ++-- .../index/query/PrefixQueryBuilder.java | 6 ++-- .../opensearch/index/query/QueryBuilder.java | 2 +- .../opensearch/index/query/QueryBuilders.java | 2 +- .../index/query/QueryRewriteContext.java | 2 +- .../index/query/QueryShardContext.java | 6 ++-- .../index/query/QueryShardException.java | 8 +++--- .../index/query/QueryStringQueryBuilder.java | 6 ++-- .../index/query/RangeQueryBuilder.java | 6 ++-- .../index/query/RegexpQueryBuilder.java | 6 ++-- .../opensearch/index/query/Rewriteable.java | 2 +- .../index/query/ScriptQueryBuilder.java | 6 ++-- .../index/query/SimpleQueryStringBuilder.java | 6 ++-- .../query/SpanContainingQueryBuilder.java | 6 ++-- .../index/query/SpanFirstQueryBuilder.java | 6 ++-- .../query/SpanMultiTermQueryBuilder.java | 6 ++-- .../index/query/SpanNearQueryBuilder.java | 6 ++-- .../index/query/SpanNotQueryBuilder.java | 6 ++-- .../index/query/SpanOrQueryBuilder.java | 6 ++-- .../index/query/SpanQueryBuilder.java | 2 +- .../index/query/SpanTermQueryBuilder.java | 4 +-- .../index/query/SpanWithinQueryBuilder.java | 6 ++-- .../index/query/TermQueryBuilder.java | 6 ++-- .../index/query/TermsQueryBuilder.java | 8 +++--- .../index/query/TermsSetQueryBuilder.java | 6 ++-- .../index/query/WildcardQueryBuilder.java | 6 ++-- .../index/query/WrapperQueryBuilder.java | 8 +++--- .../functionscore/DecayFunctionBuilder.java | 8 +++--- .../functionscore/DecayFunctionParser.java | 4 +-- .../ExponentialDecayFunctionBuilder.java | 4 +-- .../FieldValueFactorFunctionBuilder.java | 6 ++-- .../FunctionScoreQueryBuilder.java | 8 +++--- .../GaussDecayFunctionBuilder.java | 4 +-- .../LinearDecayFunctionBuilder.java | 4 +-- .../RandomScoreFunctionBuilder.java | 6 ++-- .../functionscore/ScoreFunctionBuilder.java | 6 ++-- .../ScriptScoreFunctionBuilder.java | 6 ++-- .../ScriptScoreQueryBuilder.java | 4 +-- .../query/functionscore/WeightBuilder.java | 4 +-- .../index/recovery/RecoveryStats.java | 6 ++-- .../index/refresh/RefreshStats.java | 6 ++-- .../reindex/AbstractBulkByScrollRequest.java | 4 +-- .../AbstractBulkIndexByScrollRequest.java | 4 +-- .../index/reindex/BulkByScrollResponse.java | 8 +++--- .../index/reindex/BulkByScrollTask.java | 8 +++--- .../reindex/ClientScrollableHitSource.java | 2 +- .../index/reindex/DeleteByQueryRequest.java | 2 +- .../index/reindex/ReindexRequest.java | 6 ++-- .../opensearch/index/reindex/RemoteInfo.java | 8 +++--- .../index/reindex/ScrollableHitSource.java | 10 +++---- .../index/reindex/UpdateByQueryRequest.java | 4 +-- .../RemoteRefreshSegmentPressureService.java | 2 +- .../remote/RemoteRefreshSegmentTracker.java | 8 +++--- .../opensearch/index/search/MatchQuery.java | 6 ++-- .../index/search/stats/SearchStats.java | 6 ++-- .../seqno/GlobalCheckpointSyncAction.java | 4 +-- .../index/seqno/ReplicationTracker.java | 8 +++--- .../index/seqno/RetentionLease.java | 6 ++-- .../index/seqno/RetentionLeaseActions.java | 8 +++--- .../RetentionLeaseAlreadyExistsException.java | 2 +- .../RetentionLeaseBackgroundSyncAction.java | 6 ++-- ...onLeaseInvalidRetainingSeqNoException.java | 2 +- .../RetentionLeaseNotFoundException.java | 2 +- .../index/seqno/RetentionLeaseStats.java | 6 ++-- .../index/seqno/RetentionLeaseSyncAction.java | 6 ++-- .../index/seqno/RetentionLeaseSyncer.java | 2 +- .../index/seqno/RetentionLeases.java | 6 ++-- .../opensearch/index/seqno/SeqNoStats.java | 6 ++-- .../shard/AbstractIndexShardComponent.java | 1 + .../org/opensearch/index/shard/DocsStats.java | 6 ++-- .../shard/GlobalCheckpointListeners.java | 1 + .../IllegalIndexShardStateException.java | 7 +++-- .../index/shard/IndexEventListener.java | 3 +- .../opensearch/index/shard/IndexShard.java | 5 ++-- .../shard/IndexShardClosedException.java | 3 +- .../index/shard/IndexShardComponent.java | 1 + .../IndexShardNotRecoveringException.java | 3 +- .../shard/IndexShardNotStartedException.java | 3 +- .../shard/IndexShardOperationPermits.java | 1 + .../shard/IndexShardRecoveringException.java | 3 +- .../shard/IndexShardRecoveryException.java | 3 +- .../shard/IndexShardRelocatedException.java | 3 +- .../shard/IndexShardStartedException.java | 3 +- .../shard/IndexingOperationListener.java | 1 + .../opensearch/index/shard/IndexingStats.java | 6 ++-- .../index/shard/InternalIndexingStats.java | 1 + .../index/shard/LocalShardSnapshot.java | 2 +- .../index/shard/PrimaryReplicaSyncer.java | 5 ++-- .../shard/PrimaryShardClosedException.java | 3 +- .../RemoveCorruptedShardDataCommand.java | 3 +- .../index/shard/ShardNotFoundException.java | 3 +- .../shard/ShardNotInPrimaryModeException.java | 3 +- .../org/opensearch/index/shard/ShardPath.java | 1 + .../opensearch/index/shard/ShardUtils.java | 1 + .../opensearch/index/shard/StoreRecovery.java | 3 +- .../snapshots/IndexShardRestoreException.java | 4 +-- .../IndexShardRestoreFailedException.java | 4 +-- .../BlobStoreIndexShardSnapshot.java | 2 +- .../BlobStoreIndexShardSnapshots.java | 2 +- .../stats/IndexingPressurePerShardStats.java | 6 ++-- .../index/stats/IndexingPressureStats.java | 6 ++-- .../stats/ShardIndexingPressureStats.java | 8 +++--- .../org/opensearch/index/store/Store.java | 10 +++---- .../index/store/StoreFileMetadata.java | 6 ++-- .../opensearch/index/store/StoreStats.java | 6 ++-- .../remote/filecache/FileCacheCleaner.java | 4 +-- .../remote/filecache/FileCacheStats.java | 6 ++-- .../index/termvectors/TermVectorsService.java | 2 +- .../index/translog/BaseTranslogReader.java | 2 +- .../translog/BufferedChecksumStreamInput.java | 4 +-- .../BufferedChecksumStreamOutput.java | 2 +- .../translog/InternalTranslogManager.java | 2 +- .../index/translog/NoOpTranslogManager.java | 2 +- .../index/translog/RemoteFsTranslog.java | 2 +- .../opensearch/index/translog/Translog.java | 10 +++---- .../index/translog/TranslogConfig.java | 2 +- .../translog/TranslogCorruptedException.java | 2 +- .../index/translog/TranslogException.java | 4 +-- .../index/translog/TranslogHeader.java | 6 ++-- .../index/translog/TranslogStats.java | 6 ++-- .../index/translog/TranslogWriter.java | 6 ++-- .../translog/TruncatedTranslogException.java | 2 +- .../translog/WriteOnlyTranslogManager.java | 2 +- .../CompositeTranslogEventListener.java | 2 +- .../transfer/FileTransferTracker.java | 2 +- .../transfer/TranslogTransferManager.java | 4 +-- .../index/warmer/ShardIndexWarmerService.java | 2 +- .../opensearch/index/warmer/WarmerStats.java | 6 ++-- .../AbstractIndexShardCacheEntity.java | 2 +- .../indices/AliasFilterParsingException.java | 4 +-- .../indices/IndexClosedException.java | 6 ++-- .../indices/IndexCreationException.java | 2 +- ...ndexPrimaryShardNotAllocatedException.java | 6 ++-- .../IndexTemplateMissingException.java | 6 ++-- .../indices/IndexingMemoryController.java | 2 +- .../org/opensearch/indices/IndicesModule.java | 4 +-- .../opensearch/indices/IndicesQueryCache.java | 2 +- .../indices/IndicesRequestCache.java | 2 +- .../opensearch/indices/IndicesService.java | 14 +++++----- .../indices/InvalidAliasNameException.java | 6 ++-- .../indices/InvalidIndexNameException.java | 6 ++-- .../InvalidIndexTemplateException.java | 6 ++-- .../indices/InvalidTypeNameException.java | 4 +-- .../opensearch/indices/NodeIndicesStats.java | 8 +++--- .../indices/ShardLimitValidator.java | 2 +- .../org/opensearch/indices/SystemIndices.java | 2 +- .../org/opensearch/indices/TermsLookup.java | 6 ++-- .../indices/TypeMissingException.java | 6 ++-- .../breaker/AllCircuitBreakerStats.java | 6 ++-- .../indices/breaker/CircuitBreakerStats.java | 6 ++-- .../cluster/IndicesClusterStateService.java | 4 +-- .../cache/IndicesFieldDataCache.java | 4 +-- .../recovery/DelayRecoveryException.java | 2 +- .../indices/recovery/FileChunkRequest.java | 8 +++--- .../indices/recovery/FileChunkWriter.java | 2 +- .../indices/recovery/ForceSyncRequest.java | 6 ++-- .../indices/recovery/MultiFileWriter.java | 2 +- .../recovery/PeerRecoveryNotFound.java | 4 +-- .../recovery/PeerRecoverySourceService.java | 2 +- .../recovery/PeerRecoveryTargetService.java | 4 +-- .../RecoverFilesRecoveryException.java | 6 ++-- .../recovery/RecoveryCleanFilesRequest.java | 6 ++-- .../recovery/RecoveryFailedException.java | 4 +-- .../recovery/RecoveryFilesInfoRequest.java | 6 ++-- .../RecoveryFinalizeRecoveryRequest.java | 6 ++-- .../RecoveryHandoffPrimaryContextRequest.java | 6 ++-- ...ryPrepareForTranslogOperationsRequest.java | 6 ++-- .../indices/recovery/RecoveryResponse.java | 4 +-- .../indices/recovery/RecoveryState.java | 8 +++--- .../indices/recovery/RecoveryTarget.java | 2 +- .../RecoveryTranslogOperationsRequest.java | 6 ++-- .../RecoveryTranslogOperationsResponse.java | 4 +-- .../recovery/RecoveryTransportRequest.java | 4 +-- .../recovery/ReestablishRecoveryRequest.java | 6 ++-- .../recovery/RemoteRecoveryTargetHandler.java | 6 ++-- .../recovery/RetryableTransportClient.java | 2 +- .../recovery/StartRecoveryRequest.java | 6 ++-- .../replication/CheckpointInfoRequest.java | 4 +-- .../replication/CheckpointInfoResponse.java | 4 +-- .../replication/GetSegmentFilesRequest.java | 4 +-- .../replication/GetSegmentFilesResponse.java | 4 +-- .../OngoingSegmentReplications.java | 2 +- .../PrimaryShardReplicationSource.java | 2 +- .../RemoteSegmentFileChunkWriter.java | 6 ++-- .../SegmentFileTransferHandler.java | 4 +-- .../SegmentReplicationSourceFactory.java | 2 +- .../SegmentReplicationSourceService.java | 2 +- .../replication/SegmentReplicationState.java | 6 ++-- .../replication/SegmentReplicationTarget.java | 2 +- .../SegmentReplicationTargetService.java | 2 +- .../UpdateVisibleCheckpointRequest.java | 6 ++-- .../checkpoint/PublishCheckpointAction.java | 2 +- .../checkpoint/PublishCheckpointRequest.java | 4 +-- .../checkpoint/ReplicationCheckpoint.java | 8 +++--- .../common/ReplicationCollection.java | 2 +- .../common/ReplicationFailedException.java | 4 +-- .../common/ReplicationLuceneIndex.java | 6 ++-- .../replication/common/ReplicationTarget.java | 4 +-- .../replication/common/ReplicationTimer.java | 6 ++-- .../SegmentReplicationTransportRequest.java | 4 +-- .../indices/store/IndicesStore.java | 6 ++-- .../TransportNodesListShardStoreMetadata.java | 8 +++--- .../opensearch/ingest/ConfigurationUtils.java | 2 +- .../org/opensearch/ingest/IngestInfo.java | 4 +-- .../org/opensearch/ingest/IngestMetadata.java | 4 +-- .../ingest/IngestProcessorException.java | 2 +- .../org/opensearch/ingest/IngestStats.java | 6 ++-- .../ingest/PipelineConfiguration.java | 6 ++-- .../org/opensearch/ingest/ProcessorInfo.java | 6 ++-- .../org/opensearch/monitor/fs/FsInfo.java | 6 ++-- .../org/opensearch/monitor/jvm/JvmInfo.java | 6 ++-- .../org/opensearch/monitor/jvm/JvmStats.java | 6 ++-- .../org/opensearch/monitor/os/OsInfo.java | 4 +-- .../org/opensearch/monitor/os/OsStats.java | 6 ++-- .../monitor/process/ProcessInfo.java | 4 +-- .../monitor/process/ProcessStats.java | 6 ++-- .../node/AdaptiveSelectionStats.java | 6 ++-- .../main/java/org/opensearch/node/Node.java | 2 +- .../opensearch/node/NodeClosedException.java | 2 +- .../org/opensearch/node/ReportingService.java | 2 +- .../node/ResponseCollectorService.java | 6 ++-- .../CompletionPersistentTaskAction.java | 4 +-- .../persistent/PersistentTaskParams.java | 2 +- .../persistent/PersistentTaskResponse.java | 4 +-- .../persistent/PersistentTaskState.java | 2 +- .../PersistentTasksCustomMetadata.java | 6 ++-- .../PersistentTasksNodeService.java | 4 +-- .../RemovePersistentTaskAction.java | 4 +-- .../persistent/StartPersistentTaskAction.java | 4 +-- .../UpdatePersistentTaskStatusAction.java | 4 +-- .../org/opensearch/plugins/NetworkPlugin.java | 2 +- .../java/org/opensearch/plugins/Plugin.java | 4 +-- .../org/opensearch/plugins/PluginInfo.java | 6 ++-- .../org/opensearch/plugins/SearchPlugin.java | 6 ++-- .../repositories/FilterRepository.java | 2 +- .../org/opensearch/repositories/IndexId.java | 8 +++--- .../opensearch/repositories/Repository.java | 2 +- .../repositories/RepositoryCleanupResult.java | 6 ++-- .../repositories/RepositoryData.java | 2 +- .../repositories/RepositoryException.java | 4 +-- .../repositories/RepositoryInfo.java | 6 ++-- .../RepositoryMissingException.java | 4 +-- .../repositories/RepositoryShardId.java | 6 ++-- .../repositories/RepositoryStats.java | 6 ++-- .../repositories/RepositoryStatsSnapshot.java | 6 ++-- .../RepositoryVerificationException.java | 4 +-- .../VerifyNodeRepositoryAction.java | 4 +-- .../blobstore/BlobStoreRepository.java | 10 +++---- .../blobstore/ChecksumBlobStoreFormat.java | 2 +- .../blobstore/FileRestoreContext.java | 2 +- .../org/opensearch/rest/BaseRestHandler.java | 1 + .../opensearch/rest/BytesRestResponse.java | 7 +++-- .../org/opensearch/rest/RestController.java | 15 +++++----- .../java/org/opensearch/rest/RestRequest.java | 4 +-- .../opensearch/rest/RestRequestFilter.java | 2 +- .../org/opensearch/rest/RestResponse.java | 3 +- .../opensearch/rest/action/RestActions.java | 6 ++-- .../rest/action/RestMainAction.java | 2 +- .../action/RestStatusToXContentListener.java | 2 +- .../rest/action/RestToXContentListener.java | 2 +- .../RestClusterAllocationExplainAction.java | 2 +- .../cluster/RestClusterGetSettingsAction.java | 2 +- .../admin/cluster/RestClusterStateAction.java | 2 +- .../admin/cluster/RestListTasksAction.java | 2 +- .../cluster/RestNodesHotThreadsAction.java | 2 +- .../admin/cluster/RestNodesUsageAction.java | 2 +- .../cluster/RestPutStoredScriptAction.java | 2 +- .../RestReloadSecureSettingsAction.java | 4 +-- .../RestDeleteDanglingIndexAction.java | 4 +-- .../RestImportDanglingIndexAction.java | 4 +-- .../indices/AliasesNotFoundException.java | 2 +- .../admin/indices/RestGetAliasesAction.java | 2 +- .../RestGetComponentTemplateAction.java | 6 ++-- .../RestGetComposableIndexTemplateAction.java | 6 ++-- .../indices/RestGetFieldMappingAction.java | 6 ++-- .../indices/RestGetIndexTemplateAction.java | 6 ++-- .../admin/indices/RestGetMappingAction.java | 2 +- .../indices/RestIndicesShardStoresAction.java | 2 +- .../admin/indices/RestRefreshAction.java | 2 +- .../admin/indices/RestSyncedFlushAction.java | 2 +- .../indices/RestValidateQueryAction.java | 4 +-- .../rest/action/cat/AbstractCatAction.java | 4 +-- .../rest/action/cat/RestCatAction.java | 2 +- .../opensearch/rest/action/cat/RestTable.java | 4 +-- .../rest/action/document/RestGetAction.java | 6 ++-- .../action/document/RestGetSourceAction.java | 4 +-- .../action/ingest/RestPutPipelineAction.java | 2 +- .../ingest/RestSimulatePipelineAction.java | 2 +- .../action/search/RestGetAllPitsAction.java | 2 +- .../action/search/RestMultiSearchAction.java | 4 +-- .../search/RestPutSearchPipelineAction.java | 2 +- .../rest/action/search/RestSearchAction.java | 2 +- .../script/GeneralScriptException.java | 2 +- .../java/org/opensearch/script/Script.java | 10 +++---- .../opensearch/script/ScriptCacheStats.java | 6 ++-- .../opensearch/script/ScriptContextInfo.java | 6 ++-- .../opensearch/script/ScriptContextStats.java | 6 ++-- .../opensearch/script/ScriptException.java | 6 ++-- .../script/ScriptLanguagesInfo.java | 6 ++-- .../org/opensearch/script/ScriptMetadata.java | 8 +++--- .../org/opensearch/script/ScriptStats.java | 6 ++-- .../org/opensearch/script/ScriptType.java | 6 ++-- .../opensearch/script/StoredScriptSource.java | 10 +++---- .../org/opensearch/search/DocValueFormat.java | 6 ++-- .../org/opensearch/search/MultiValueMode.java | 6 ++-- .../org/opensearch/search/RescoreDocIds.java | 6 ++-- .../java/org/opensearch/search/Scroll.java | 6 ++-- .../search/SearchContextMissingException.java | 6 ++-- .../opensearch/search/SearchException.java | 4 +-- .../opensearch/search/SearchExtBuilder.java | 8 +++--- .../java/org/opensearch/search/SearchHit.java | 18 ++++++------ .../org/opensearch/search/SearchHits.java | 8 +++--- .../org/opensearch/search/SearchModule.java | 6 ++-- .../search/SearchParseException.java | 6 ++-- .../opensearch/search/SearchPhaseResult.java | 4 +-- .../org/opensearch/search/SearchService.java | 8 +++--- .../opensearch/search/SearchShardTarget.java | 10 +++---- .../opensearch/search/SearchSortValues.java | 8 +++--- .../search/SearchSortValuesAndFormats.java | 6 ++-- .../AbstractAggregationBuilder.java | 4 +-- .../aggregations/AggregationBuilder.java | 2 +- .../AggregationExecutionException.java | 2 +- .../AggregationInitializationException.java | 2 +- .../search/aggregations/Aggregations.java | 4 +-- .../search/aggregations/Aggregator.java | 6 ++-- .../aggregations/AggregatorFactories.java | 8 +++--- .../search/aggregations/BucketOrder.java | 4 +-- .../aggregations/InternalAggregation.java | 6 ++-- .../aggregations/InternalAggregations.java | 6 ++-- .../InternalMultiBucketAggregation.java | 4 +-- .../search/aggregations/InternalOrder.java | 6 ++-- .../InvalidAggregationPathException.java | 2 +- .../MultiBucketConsumerService.java | 6 ++-- .../ParsedMultiBucketAggregation.java | 4 +-- .../PipelineAggregationBuilder.java | 2 +- .../InternalSingleBucketAggregation.java | 4 +-- .../bucket/ParsedSingleBucketAggregation.java | 4 +-- .../AdjacencyMatrixAggregationBuilder.java | 4 +-- .../adjacency/AdjacencyMatrixAggregator.java | 6 ++-- .../adjacency/InternalAdjacencyMatrix.java | 4 +-- .../CompositeAggregationBuilder.java | 6 ++-- .../bucket/composite/CompositeKey.java | 6 ++-- .../CompositeValuesSourceBuilder.java | 6 ++-- .../CompositeValuesSourceParserHelper.java | 8 +++--- .../DateHistogramValuesSourceBuilder.java | 4 +-- .../HistogramValuesSourceBuilder.java | 4 +-- .../bucket/composite/InternalComposite.java | 4 +-- .../composite/TermsValuesSourceBuilder.java | 4 +-- .../filter/FilterAggregationBuilder.java | 4 +-- .../filter/FiltersAggregationBuilder.java | 6 ++-- .../bucket/filter/FiltersAggregator.java | 6 ++-- .../bucket/filter/InternalFilter.java | 2 +- .../bucket/filter/InternalFilters.java | 4 +-- .../bucket/filter/ParsedFilters.java | 4 +-- .../global/GlobalAggregationBuilder.java | 4 +-- .../bucket/global/InternalGlobal.java | 2 +- .../AutoDateHistogramAggregationBuilder.java | 6 ++-- .../AutoDateHistogramAggregator.java | 2 +- .../DateHistogramAggregationBuilder.java | 4 +-- .../histogram/DateHistogramInterval.java | 6 ++-- .../bucket/histogram/DateIntervalWrapper.java | 6 ++-- .../bucket/histogram/DoubleBounds.java | 6 ++-- .../HistogramAggregationBuilder.java | 4 +-- .../histogram/InternalAutoDateHistogram.java | 4 +-- .../histogram/InternalDateHistogram.java | 4 +-- .../bucket/histogram/InternalHistogram.java | 4 +-- .../InternalVariableWidthHistogram.java | 4 +-- .../bucket/histogram/LongBounds.java | 6 ++-- .../ParsedVariableWidthHistogram.java | 4 +-- ...iableWidthHistogramAggregationBuilder.java | 4 +-- .../bucket/missing/InternalMissing.java | 2 +- .../missing/MissingAggregationBuilder.java | 4 +-- .../bucket/missing/MissingOrder.java | 6 ++-- .../bucket/nested/InternalNested.java | 2 +- .../bucket/nested/InternalReverseNested.java | 2 +- .../nested/NestedAggregationBuilder.java | 6 ++-- .../ReverseNestedAggregationBuilder.java | 6 ++-- .../bucket/range/AbstractRangeBuilder.java | 6 ++-- .../range/DateRangeAggregationBuilder.java | 2 +- .../range/GeoDistanceAggregationBuilder.java | 8 +++--- .../bucket/range/InternalBinaryRange.java | 4 +-- .../bucket/range/InternalDateRange.java | 2 +- .../bucket/range/InternalGeoDistance.java | 2 +- .../bucket/range/InternalRange.java | 4 +-- .../range/IpRangeAggregationBuilder.java | 6 ++-- .../bucket/range/ParsedBinaryRange.java | 4 +-- .../bucket/range/ParsedRange.java | 4 +-- .../bucket/range/RangeAggregationBuilder.java | 2 +- .../bucket/range/RangeAggregator.java | 6 ++-- .../DiversifiedAggregationBuilder.java | 4 +-- .../bucket/sampler/InternalSampler.java | 2 +- .../sampler/SamplerAggregationBuilder.java | 6 ++-- .../bucket/sampler/UnmappedSampler.java | 2 +- .../bucket/terms/DoubleTerms.java | 4 +-- .../GlobalOrdinalsStringTermsAggregator.java | 2 +- .../bucket/terms/IncludeExclude.java | 6 ++-- .../bucket/terms/InternalMappedRareTerms.java | 4 +-- .../terms/InternalMappedSignificantTerms.java | 4 +-- .../bucket/terms/InternalMappedTerms.java | 4 +-- .../bucket/terms/InternalMultiTerms.java | 4 +-- .../bucket/terms/InternalRareTerms.java | 4 +-- .../terms/InternalSignificantTerms.java | 4 +-- .../bucket/terms/InternalTerms.java | 4 +-- .../bucket/terms/LongRareTerms.java | 4 +-- .../aggregations/bucket/terms/LongTerms.java | 4 +-- .../terms/MultiTermsAggregationBuilder.java | 4 +-- .../bucket/terms/MultiTermsAggregator.java | 6 ++-- .../bucket/terms/ParsedRareTerms.java | 2 +- .../bucket/terms/ParsedSignificantTerms.java | 2 +- .../bucket/terms/ParsedTerms.java | 2 +- .../terms/RareTermsAggregationBuilder.java | 4 +-- .../bucket/terms/SignificantLongTerms.java | 4 +-- .../bucket/terms/SignificantStringTerms.java | 4 +-- .../SignificantTermsAggregationBuilder.java | 4 +-- .../SignificantTextAggregationBuilder.java | 4 +-- .../bucket/terms/StringRareTerms.java | 4 +-- .../bucket/terms/StringTerms.java | 4 +-- .../bucket/terms/TermsAggregationBuilder.java | 4 +-- .../bucket/terms/TermsAggregator.java | 6 ++-- .../bucket/terms/UnmappedRareTerms.java | 4 +-- .../terms/UnmappedSignificantTerms.java | 4 +-- .../bucket/terms/UnmappedTerms.java | 4 +-- .../bucket/terms/UnsignedLongTerms.java | 4 +-- .../bucket/terms/heuristic/ChiSquare.java | 2 +- .../bucket/terms/heuristic/GND.java | 4 +-- .../bucket/terms/heuristic/JLHScore.java | 4 +-- .../terms/heuristic/MutualInformation.java | 2 +- .../heuristic/NXYSignificanceHeuristic.java | 4 +-- .../terms/heuristic/PercentageScore.java | 4 +-- .../terms/heuristic/ScriptHeuristic.java | 4 +-- .../heuristic/SignificanceHeuristic.java | 2 +- .../metrics/AbstractHyperLogLogPlusPlus.java | 4 +-- .../AbstractInternalHDRPercentiles.java | 4 +-- .../AbstractInternalTDigestPercentiles.java | 4 +-- ...AbstractPercentilesAggregationBuilder.java | 6 ++-- .../metrics/AvgAggregationBuilder.java | 4 +-- .../CardinalityAggregationBuilder.java | 4 +-- .../ExtendedStatsAggregationBuilder.java | 4 +-- .../GeoCentroidAggregationBuilder.java | 4 +-- .../metrics/HyperLogLogPlusPlus.java | 2 +- .../aggregations/metrics/InternalAvg.java | 4 +-- .../metrics/InternalCardinality.java | 4 +-- .../metrics/InternalExtendedStats.java | 4 +-- .../metrics/InternalGeoCentroid.java | 4 +-- .../metrics/InternalHDRPercentileRanks.java | 2 +- .../metrics/InternalHDRPercentiles.java | 2 +- .../aggregations/metrics/InternalMax.java | 4 +-- .../InternalMedianAbsoluteDeviation.java | 4 +-- .../aggregations/metrics/InternalMin.java | 4 +-- .../InternalNumericMetricsAggregation.java | 2 +- .../metrics/InternalScriptedMetric.java | 4 +-- .../aggregations/metrics/InternalStats.java | 4 +-- .../aggregations/metrics/InternalSum.java | 4 +-- .../InternalTDigestPercentileRanks.java | 2 +- .../metrics/InternalTDigestPercentiles.java | 2 +- .../aggregations/metrics/InternalTopHits.java | 4 +-- .../metrics/InternalValueCount.java | 4 +-- .../metrics/InternalWeightedAvg.java | 4 +-- .../metrics/MaxAggregationBuilder.java | 4 +-- ...anAbsoluteDeviationAggregationBuilder.java | 4 +-- .../metrics/MinAggregationBuilder.java | 4 +-- .../metrics/ParsedScriptedMetric.java | 2 +- .../PercentileRanksAggregationBuilder.java | 2 +- .../PercentilesAggregationBuilder.java | 2 +- .../metrics/PercentilesConfig.java | 6 ++-- .../metrics/PercentilesMethod.java | 6 ++-- .../ScriptedMetricAggregationBuilder.java | 4 +-- .../metrics/ScriptedMetricAggregator.java | 2 +- .../metrics/StatsAggregationBuilder.java | 4 +-- .../metrics/SumAggregationBuilder.java | 4 +-- .../aggregations/metrics/TDigestState.java | 4 +-- .../metrics/TopHitsAggregationBuilder.java | 6 ++-- .../metrics/ValueCountAggregationBuilder.java | 4 +-- .../WeightedAvgAggregationBuilder.java | 4 +-- .../AbstractPipelineAggregationBuilder.java | 4 +-- .../AvgBucketPipelineAggregationBuilder.java | 4 +-- .../aggregations/pipeline/BucketHelpers.java | 8 +++--- .../pipeline/BucketMetricsParser.java | 2 +- ...cketMetricsPipelineAggregationBuilder.java | 4 +-- ...ucketScriptPipelineAggregationBuilder.java | 4 +-- ...ketSelectorPipelineAggregationBuilder.java | 6 ++-- .../BucketSortPipelineAggregationBuilder.java | 4 +-- ...mulativeSumPipelineAggregationBuilder.java | 4 +-- .../DerivativePipelineAggregationBuilder.java | 6 ++-- .../aggregations/pipeline/EwmaModel.java | 4 +-- ...StatsBucketPipelineAggregationBuilder.java | 4 +-- .../pipeline/HoltLinearModel.java | 4 +-- .../pipeline/HoltWintersModel.java | 4 +-- .../pipeline/InternalBucketMetricValue.java | 4 +-- .../pipeline/InternalDerivative.java | 4 +-- .../pipeline/InternalExtendedStatsBucket.java | 2 +- .../pipeline/InternalPercentilesBucket.java | 4 +-- .../pipeline/InternalSimpleValue.java | 4 +-- .../pipeline/InternalStatsBucket.java | 2 +- .../aggregations/pipeline/LinearModel.java | 4 +-- .../MaxBucketPipelineAggregationBuilder.java | 4 +-- .../MinBucketPipelineAggregationBuilder.java | 4 +-- .../aggregations/pipeline/MovAvgModel.java | 4 +-- .../MovAvgPipelineAggregationBuilder.java | 6 ++-- .../MovFnPipelineAggregationBuilder.java | 4 +-- ...tilesBucketPipelineAggregationBuilder.java | 4 +-- .../SerialDiffPipelineAggregationBuilder.java | 6 ++-- .../aggregations/pipeline/SimpleModel.java | 4 +-- ...StatsBucketPipelineAggregationBuilder.java | 4 +-- .../SumBucketPipelineAggregationBuilder.java | 4 +-- .../aggregations/support/AggregationInfo.java | 4 +-- .../BaseMultiValuesSourceFieldConfig.java | 6 ++-- .../support/MultiTermsValuesSourceConfig.java | 4 +-- .../MultiValuesSourceAggregationBuilder.java | 4 +-- .../support/MultiValuesSourceFieldConfig.java | 4 +-- .../support/MultiValuesSourceParseHelper.java | 2 +- .../aggregations/support/ValueType.java | 6 ++-- .../ValuesSourceAggregationBuilder.java | 4 +-- .../stats/SearchBackpressureStats.java | 6 ++-- .../stats/SearchShardTaskStats.java | 6 ++-- .../backpressure/stats/SearchTaskStats.java | 6 ++-- .../trackers/CpuUsageTracker.java | 4 +-- .../trackers/ElapsedTimeTracker.java | 4 +-- .../trackers/HeapUsageTracker.java | 4 +-- .../trackers/TaskResourceUsageTracker.java | 2 +- .../search/builder/PointInTimeBuilder.java | 6 ++-- .../search/builder/SearchSourceBuilder.java | 8 +++--- .../builder/SearchSourceBuilderException.java | 2 +- .../search/collapse/CollapseBuilder.java | 8 +++--- .../opensearch/search/dfs/AggregatedDfs.java | 6 ++-- .../dfs/DfsPhaseExecutionException.java | 2 +- .../search/dfs/DfsSearchResult.java | 4 +-- .../opensearch/search/fetch/FetchPhase.java | 2 +- .../fetch/FetchPhaseExecutionException.java | 2 +- .../search/fetch/FetchSearchResult.java | 4 +-- .../search/fetch/QueryFetchSearchResult.java | 4 +-- .../fetch/ScrollQueryFetchSearchResult.java | 4 +-- .../search/fetch/ShardFetchRequest.java | 4 +-- .../search/fetch/ShardFetchSearchRequest.java | 4 +-- .../search/fetch/StoredFieldsContext.java | 8 +++--- .../fetch/subphase/FetchSourceContext.java | 8 +++--- .../fetch/subphase/FetchSourcePhase.java | 2 +- .../search/fetch/subphase/FieldAndFormat.java | 6 ++-- .../highlight/AbstractHighlighterBuilder.java | 8 +++--- .../highlight/FastVectorHighlighter.java | 2 +- .../subphase/highlight/HighlightBuilder.java | 6 ++-- .../subphase/highlight/HighlightField.java | 12 ++++---- .../subphase/highlight/PlainHighlighter.java | 2 +- .../highlight/UnifiedHighlighter.java | 2 +- .../search/internal/AliasFilter.java | 6 ++-- .../internal/InternalScrollSearchRequest.java | 4 +-- .../internal/InternalSearchResponse.java | 6 ++-- .../search/internal/ShardSearchContextId.java | 6 ++-- .../search/internal/ShardSearchRequest.java | 10 +++---- .../search/lookup/SourceLookup.java | 2 +- .../opensearch/search/pipeline/Pipeline.java | 6 ++-- .../pipeline/PipelineConfiguration.java | 6 ++-- .../search/pipeline/PipelineWithMetrics.java | 2 +- .../search/pipeline/ProcessorInfo.java | 6 ++-- .../search/pipeline/SearchPipelineInfo.java | 4 +-- .../pipeline/SearchPipelineMetadata.java | 4 +-- .../SearchPipelineProcessingException.java | 2 +- .../pipeline/SearchPipelineService.java | 2 +- .../search/pipeline/SearchPipelineStats.java | 6 ++-- .../search/profile/NetworkTime.java | 6 ++-- .../search/profile/ProfileResult.java | 6 ++-- .../search/profile/ProfileShardResult.java | 6 ++-- .../profile/SearchProfileShardResults.java | 8 +++--- .../AggregationProfileShardResult.java | 8 +++--- .../search/profile/query/CollectorResult.java | 8 +++--- .../query/QueryProfileShardResult.java | 8 +++--- .../query/QueryPhaseExecutionException.java | 2 +- .../search/query/QuerySearchRequest.java | 4 +-- .../search/query/QuerySearchResult.java | 4 +-- .../search/query/ScrollQuerySearchResult.java | 4 +-- .../search/rescore/QueryRescoreMode.java | 6 ++-- .../search/rescore/QueryRescorerBuilder.java | 6 ++-- .../search/rescore/RescorerBuilder.java | 8 +++--- .../searchafter/SearchAfterBuilder.java | 10 +++---- .../opensearch/search/slice/SliceBuilder.java | 6 ++-- .../opensearch/search/sort/BucketedSort.java | 2 +- .../search/sort/FieldSortBuilder.java | 6 ++-- .../search/sort/GeoDistanceSortBuilder.java | 6 ++-- .../org/opensearch/search/sort/MinAndMax.java | 6 ++-- .../search/sort/NestedSortBuilder.java | 6 ++-- .../search/sort/ScoreSortBuilder.java | 4 +-- .../search/sort/ScriptSortBuilder.java | 6 ++-- .../opensearch/search/sort/SortBuilder.java | 4 +-- .../org/opensearch/search/sort/SortMode.java | 6 ++-- .../org/opensearch/search/sort/SortOrder.java | 6 ++-- .../org/opensearch/search/sort/SortValue.java | 8 +++--- .../org/opensearch/search/suggest/SortBy.java | 6 ++-- .../opensearch/search/suggest/Suggest.java | 16 +++++------ .../search/suggest/SuggestBuilder.java | 8 +++--- .../search/suggest/SuggestionBuilder.java | 8 +++--- .../suggest/completion/CompletionStats.java | 6 ++-- .../completion/CompletionSuggester.java | 2 +- .../completion/CompletionSuggestion.java | 8 +++--- .../CompletionSuggestionBuilder.java | 6 ++-- .../suggest/completion/FuzzyOptions.java | 6 ++-- .../suggest/completion/RegexOptions.java | 6 ++-- .../DirectCandidateGeneratorBuilder.java | 4 +-- .../search/suggest/phrase/Laplace.java | 4 +-- .../suggest/phrase/LinearInterpolation.java | 6 ++-- .../suggest/phrase/PhraseSuggester.java | 2 +- .../suggest/phrase/PhraseSuggestion.java | 6 ++-- .../phrase/PhraseSuggestionBuilder.java | 8 +++--- .../search/suggest/phrase/SmoothingModel.java | 4 +-- .../search/suggest/phrase/StupidBackoff.java | 4 +-- .../search/suggest/term/TermSuggester.java | 4 +-- .../search/suggest/term/TermSuggestion.java | 6 ++-- .../suggest/term/TermSuggestionBuilder.java | 8 +++--- .../ConcurrentSnapshotExecutionException.java | 4 +-- .../InFlightShardSnapshotStates.java | 2 +- .../InternalSnapshotsInfoService.java | 2 +- .../InvalidSnapshotNameException.java | 4 +-- .../org/opensearch/snapshots/RestoreInfo.java | 8 +++--- .../opensearch/snapshots/RestoreService.java | 4 +-- .../org/opensearch/snapshots/Snapshot.java | 6 ++-- .../snapshots/SnapshotCreationException.java | 2 +- .../snapshots/SnapshotException.java | 4 +-- .../org/opensearch/snapshots/SnapshotId.java | 6 ++-- .../SnapshotInProgressException.java | 4 +-- .../SnapshotInUseDeletionException.java | 4 +-- .../opensearch/snapshots/SnapshotInfo.java | 12 ++++---- .../snapshots/SnapshotMissingException.java | 4 +-- .../snapshots/SnapshotRestoreException.java | 2 +- .../snapshots/SnapshotShardFailure.java | 12 ++++---- .../snapshots/SnapshotShardsService.java | 6 ++-- .../snapshots/SnapshotsService.java | 6 ++-- ...UpdateIndexShardSnapshotStatusRequest.java | 6 ++-- ...pdateIndexShardSnapshotStatusResponse.java | 2 +- .../opensearch/snapshots/package-info.java | 2 +- .../org/opensearch/tasks/RawTaskStatus.java | 6 ++-- .../SearchShardTaskCancellationStats.java | 6 ++-- .../main/java/org/opensearch/tasks/Task.java | 2 +- .../tasks/TaskCancellationService.java | 4 +-- .../tasks/TaskCancellationStats.java | 6 ++-- .../tasks/TaskCancelledException.java | 2 +- .../java/org/opensearch/tasks/TaskId.java | 6 ++-- .../java/org/opensearch/tasks/TaskInfo.java | 8 +++--- .../opensearch/tasks/TaskResourceStats.java | 6 ++-- .../opensearch/tasks/TaskResourceUsage.java | 6 ++-- .../java/org/opensearch/tasks/TaskResult.java | 8 +++--- .../org/opensearch/tasks/TaskThreadUsage.java | 6 ++-- .../org/opensearch/threadpool/ThreadPool.java | 6 ++-- .../opensearch/threadpool/ThreadPoolInfo.java | 4 +-- .../threadpool/ThreadPoolStats.java | 6 ++-- .../ActionNotFoundTransportException.java | 4 +-- .../transport/ActionTransportException.java | 4 +-- .../transport/BindTransportException.java | 2 +- .../transport/BytesTransportRequest.java | 6 ++-- .../CompressibleBytesOutputStream.java | 6 ++-- .../transport/ConnectTransportException.java | 4 +-- .../EmptyTransportResponseHandler.java | 2 +- .../java/org/opensearch/transport/Header.java | 2 +- .../transport/InboundAggregator.java | 6 ++-- .../opensearch/transport/InboundDecoder.java | 4 +-- .../opensearch/transport/InboundHandler.java | 8 +++--- .../opensearch/transport/InboundMessage.java | 2 +- .../opensearch/transport/InboundPipeline.java | 2 +- .../opensearch/transport/NetworkMessage.java | 2 +- .../transport/NoSeedNodeLeftException.java | 2 +- .../NoSuchRemoteClusterException.java | 2 +- .../transport/NodeDisconnectedException.java | 2 +- .../transport/NodeNotConnectedException.java | 2 +- .../NodeShouldNotConnectException.java | 2 +- .../NotSerializableTransportException.java | 2 +- .../opensearch/transport/OutboundHandler.java | 2 +- .../opensearch/transport/OutboundMessage.java | 10 +++---- .../transport/PlainTransportFuture.java | 2 +- .../transport/ProxyConnectionStrategy.java | 6 ++-- .../ReceiveTimeoutTransportException.java | 2 +- .../transport/RemoteClusterConnection.java | 2 +- .../transport/RemoteConnectionInfo.java | 6 ++-- .../transport/RemoteConnectionStrategy.java | 2 +- .../transport/RemoteTransportException.java | 2 +- .../transport/RequestHandlerRegistry.java | 4 +-- ...ponseHandlerFailureTransportException.java | 2 +- .../SendRequestTransportException.java | 2 +- .../transport/SniffConnectionStrategy.java | 6 ++-- .../org/opensearch/transport/TcpChannel.java | 2 +- .../org/opensearch/transport/TcpHeader.java | 2 +- .../opensearch/transport/TcpTransport.java | 10 +++---- .../transport/TransportActionProxy.java | 6 ++-- .../transport/TransportDecompressor.java | 4 +-- .../transport/TransportException.java | 2 +- .../transport/TransportHandshaker.java | 6 ++-- .../opensearch/transport/TransportInfo.java | 4 +-- .../transport/TransportInterceptor.java | 2 +- .../transport/TransportKeepAlive.java | 2 +- .../opensearch/transport/TransportLogger.java | 6 ++-- .../transport/TransportMessage.java | 4 +-- .../transport/TransportRequest.java | 4 +-- .../transport/TransportResponse.java | 4 +-- .../transport/TransportResponseHandler.java | 4 +-- .../TransportSerializationException.java | 2 +- .../transport/TransportService.java | 6 ++-- .../opensearch/transport/TransportStats.java | 6 ++-- .../test/java/org/opensearch/BuildTests.java | 6 ++-- .../ExceptionSerializationTests.java | 24 ++++++++-------- .../org/opensearch/ExceptionsHelperTests.java | 10 +++---- .../opensearch/OpenSearchExceptionTests.java | 12 ++++---- .../action/DocWriteResponseTests.java | 4 +-- .../action/DynamicActionRegistryTests.java | 4 +-- .../action/OriginalIndicesTests.java | 2 +- .../RenamedTimeoutRequestParameterTests.java | 2 +- .../ShardOperationFailedExceptionTests.java | 5 ++-- .../ShardValidateQueryRequestTests.java | 8 +++--- .../ClusterAllocationExplanationTests.java | 6 ++-- ...tAddVotingConfigExclusionsActionTests.java | 2 +- ...learVotingConfigExclusionsActionTests.java | 2 +- .../DeleteDecommissionStateRequestTests.java | 2 +- .../health/ClusterHealthRequestTests.java | 2 +- .../health/ClusterHealthResponsesTests.java | 6 ++-- .../TransportClusterHealthActionTests.java | 4 +-- .../node/info/NodesInfoRequestTests.java | 2 +- .../cluster/node/stats/NodeStatsTests.java | 2 +- .../node/stats/NodesStatsRequestTests.java | 2 +- .../node/tasks/CancellableTasksTests.java | 4 +-- .../node/tasks/ResourceAwareTasksTests.java | 4 +-- .../node/tasks/TaskManagerTestCase.java | 8 +++--- .../admin/cluster/node/tasks/TaskTests.java | 2 +- .../cluster/node/tasks/TestTaskPlugin.java | 8 +++--- .../node/tasks/TransportTasksActionTests.java | 8 +++--- .../RestoreRemoteStoreRequestTests.java | 4 +-- .../stats/RemoteStoreStatsRequestTests.java | 2 +- .../stats/RemoteStoreStatsResponseTests.java | 6 ++-- .../stats/RemoteStoreStatsTestHelper.java | 2 +- .../stats/RemoteStoreStatsTests.java | 6 ++-- .../TransportRemoteStoreStatsActionTests.java | 2 +- .../put/PutRepositoryRequestTests.java | 2 +- .../reroute/ClusterRerouteRequestTests.java | 8 +++--- .../cluster/reroute/ClusterRerouteTests.java | 8 +++--- .../ClusterUpdateSettingsRequestTests.java | 2 +- .../ClusterUpdateSettingsResponseTests.java | 2 +- .../ClusterSearchShardsRequestTests.java | 2 +- .../ClusterSearchShardsResponseTests.java | 8 +++--- ...ClusterPutWeightedRoutingRequestTests.java | 2 +- .../create/CreateSnapshotRequestTests.java | 2 +- .../create/CreateSnapshotResponseTests.java | 2 +- .../get/GetSnapshotsResponseTests.java | 4 +-- .../restore/RestoreSnapshotRequestTests.java | 4 +-- .../status/SnapshotIndexShardStatusTests.java | 6 ++-- .../status/SnapshotIndexStatusTests.java | 2 +- .../snapshots/status/SnapshotStatusTests.java | 2 +- .../state/ClusterStateRequestTests.java | 2 +- .../state/ClusterStateResponseTests.java | 4 +-- .../cluster/stats/AnalysisStatsTests.java | 2 +- .../cluster/stats/MappingStatsTests.java | 2 +- .../GetScriptContextResponseTests.java | 2 +- .../GetScriptLanguageResponseTests.java | 2 +- .../GetStoredScriptRequestTests.java | 2 +- .../GetStoredScriptResponseTests.java | 2 +- .../PutStoredScriptRequestTests.java | 6 ++-- .../ScriptContextInfoSerializingTests.java | 2 +- .../ScriptMethodInfoSerializingTests.java | 2 +- .../ScriptParameterInfoSerializingTests.java | 2 +- .../indices/alias/AliasActionsTests.java | 4 +-- .../alias/IndicesAliasesRequestTests.java | 2 +- .../alias/get/GetAliasesResponseTests.java | 2 +- .../indices/analyze/AnalyzeRequestTests.java | 2 +- .../indices/analyze/AnalyzeResponseTests.java | 4 +-- .../clear/ClearIndicesCacheResponseTests.java | 2 +- ...TransportClearIndicesCacheActionTests.java | 4 +-- .../indices/close/CloseIndexRequestTests.java | 2 +- .../close/CloseIndexResponseTests.java | 6 ++-- ...portVerifyShardBeforeCloseActionTests.java | 2 +- .../create/CreateIndexRequestTests.java | 2 +- .../create/CreateIndexResponseTests.java | 2 +- .../CreateDataStreamRequestTests.java | 2 +- .../DataStreamsStatsResponseTests.java | 4 +-- .../DeleteDataStreamRequestTests.java | 4 +-- .../GetDataStreamsRequestTests.java | 2 +- .../GetDataStreamsResponseTests.java | 2 +- .../indices/flush/FlushResponseTests.java | 2 +- .../forcemerge/ForceMergeResponseTests.java | 2 +- .../forcemerge/RestForceMergeActionTests.java | 2 +- .../indices/get/GetIndexActionTests.java | 2 +- .../indices/get/GetIndexResponseTests.java | 2 +- .../get/GetFieldMappingsResponseTests.java | 6 ++-- .../mapping/get/GetMappingsResponseTests.java | 2 +- .../mapping/put/PutMappingRequestTests.java | 4 +-- .../indices/open/OpenIndexRequestTests.java | 2 +- .../indices/open/OpenIndexResponseTests.java | 2 +- .../indices/refresh/RefreshResponseTests.java | 2 +- .../resolve/ResolveIndexRequestTests.java | 2 +- .../resolve/ResolveIndexResponseTests.java | 2 +- .../MetadataRolloverServiceTests.java | 2 +- .../rollover/RolloverRequestTests.java | 10 +++---- .../rollover/RolloverResponseTests.java | 2 +- .../TransportRolloverActionTests.java | 2 +- .../settings/get/GetSettingsActionTests.java | 2 +- .../settings/get/GetSettingsRequestTests.java | 4 +-- .../get/GetSettingsResponseTests.java | 2 +- ...dateSettingsRequestSerializationTests.java | 2 +- .../IndicesShardStoreResponseTests.java | 2 +- .../indices/shrink/ResizeRequestTests.java | 2 +- .../indices/shrink/ResizeResponseTests.java | 2 +- .../stats/IndicesStatsResponseTests.java | 4 +-- .../indices/stats/IndicesStatsTests.java | 2 +- ...teComposableIndexTemplateRequestTests.java | 2 +- .../GetComponentTemplateResponseTests.java | 2 +- ...etComposableIndexTemplateRequestTests.java | 2 +- ...tComposableIndexTemplateResponseTests.java | 2 +- .../get/GetIndexTemplatesResponseTests.java | 2 +- .../SimulateIndexTemplateRequestTests.java | 2 +- .../post/SimulateTemplateRequestTests.java | 2 +- ...utComposableIndexTemplateRequestTests.java | 2 +- .../validate/query/QueryExplanationTests.java | 2 +- .../query/ValidateQueryResponseTests.java | 2 +- .../action/bulk/BulkItemResponseTests.java | 2 +- .../BulkPrimaryExecutionContextTests.java | 2 +- .../action/bulk/BulkProcessorTests.java | 2 +- .../action/bulk/BulkRequestModifierTests.java | 2 +- .../action/bulk/BulkRequestParserTests.java | 2 +- .../action/bulk/BulkRequestTests.java | 6 ++-- .../action/bulk/BulkResponseTests.java | 2 +- .../action/bulk/BulkShardRequestTests.java | 2 +- .../opensearch/action/bulk/RetryTests.java | 2 +- .../bulk/TransportShardBulkActionTests.java | 6 ++-- .../action/delete/DeleteResponseTests.java | 4 +-- .../action/explain/ExplainRequestTests.java | 6 ++-- .../action/explain/ExplainResponseTests.java | 6 ++-- .../FieldCapabilitiesRequestTests.java | 2 +- .../FieldCapabilitiesResponseTests.java | 2 +- .../fieldcaps/FieldCapabilitiesTests.java | 2 +- .../MergedFieldCapabilitiesResponseTests.java | 4 +-- .../action/get/GetResponseTests.java | 6 ++-- .../action/get/MultiGetRequestTests.java | 4 +-- .../action/get/MultiGetResponseTests.java | 2 +- .../action/get/MultiGetShardRequestTests.java | 2 +- .../get/TransportMultiGetActionTests.java | 6 ++-- .../action/index/IndexRequestTests.java | 8 +++--- .../action/index/IndexResponseTests.java | 4 +-- .../ingest/GetPipelineResponseTests.java | 4 +-- .../ingest/PutPipelineRequestTests.java | 6 ++-- .../SimulateDocumentBaseResultTests.java | 2 +- .../ingest/SimulatePipelineRequestTests.java | 4 +-- .../ingest/SimulatePipelineResponseTests.java | 2 +- .../ingest/SimulateProcessorResultTests.java | 2 +- .../ingest/WriteableIngestDocumentTests.java | 4 +-- .../action/main/MainActionTests.java | 2 +- .../action/main/MainResponseTests.java | 2 +- .../resync/ResyncReplicationRequestTests.java | 6 ++-- ...TransportResyncReplicationActionTests.java | 6 ++-- .../AbstractSearchAsyncActionTests.java | 4 +-- .../CanMatchPreFilterSearchPhaseTests.java | 2 +- .../search/ClearScrollControllerTests.java | 2 +- .../search/ClearScrollRequestTests.java | 2 +- .../action/search/CountedCollectorTests.java | 2 +- .../search/CreatePitControllerTests.java | 2 +- .../action/search/DfsQueryPhaseTests.java | 2 +- .../action/search/FetchSearchPhaseTests.java | 2 +- .../search/MultiSearchRequestTests.java | 6 ++-- .../action/search/PitTestsUtil.java | 2 +- .../search/QueryPhaseResultConsumerTests.java | 2 +- .../action/search/SearchAsyncActionTests.java | 6 ++-- .../action/search/SearchContextIdTests.java | 4 +-- .../search/SearchPhaseControllerTests.java | 6 ++-- .../SearchPhaseExecutionExceptionTests.java | 8 +++--- .../SearchQueryThenFetchAsyncActionTests.java | 2 +- .../search/SearchResponseMergerTests.java | 6 ++-- .../action/search/SearchResponseTests.java | 4 +-- .../search/SearchScrollAsyncActionTests.java | 2 +- .../search/SearchScrollRequestTests.java | 4 +-- .../search/SearchShardIteratorTests.java | 2 +- .../action/search/SearchShardTests.java | 4 +-- .../search/ShardSearchFailureTests.java | 8 +++--- .../search/TransportDeletePitActionTests.java | 2 +- .../search/TransportSearchActionTests.java | 6 ++-- .../search/TransportSearchHelperTests.java | 2 +- .../action/support/ActiveShardCountTests.java | 2 +- .../action/support/IndicesOptionsTests.java | 4 +-- .../action/support/RefreshPolicyTests.java | 2 +- .../TransportActionFilterChainTests.java | 2 +- .../TransportBroadcastByNodeActionTests.java | 12 ++++---- .../ShardsAcknowledgedResponseTests.java | 6 ++-- ...ransportClusterManagerNodeActionTests.java | 6 ++-- .../nodes/TransportNodesActionTests.java | 6 ++-- .../BroadcastReplicationTests.java | 10 +++---- .../PendingReplicationActionsTests.java | 2 +- .../ReplicationOperationTests.java | 2 +- .../replication/ReplicationResponseTests.java | 6 ++-- .../TransportReplicationActionTests.java | 12 ++++---- ...ReplicationAllPermitsAcquisitionTests.java | 6 ++-- ...rtWriteActionForIndexingPressureTests.java | 6 ++-- .../TransportWriteActionTests.java | 6 ++-- ...ortInstanceSingleOperationActionTests.java | 10 +++---- .../termvectors/TermVectorsUnitTests.java | 10 +++---- .../TransportMultiTermVectorsActionTests.java | 6 ++-- .../action/update/UpdateRequestTests.java | 8 +++--- .../action/update/UpdateResponseTests.java | 6 ++-- .../client/node/NodeClientHeadersTests.java | 2 +- .../cluster/ClusterChangedEventTests.java | 2 +- .../opensearch/cluster/ClusterInfoTests.java | 2 +- .../opensearch/cluster/ClusterStateTests.java | 6 ++-- .../opensearch/cluster/DiskUsageTests.java | 4 +-- .../opensearch/cluster/FeatureAwareTests.java | 2 +- .../index/MappingUpdatedActionTests.java | 2 +- ...rdFailedClusterStateTaskExecutorTests.java | 4 +-- ...dStartedClusterStateTaskExecutorTests.java | 2 +- .../action/shard/ShardStateActionTests.java | 8 +++--- ...terAwarenessAttributeValueHealthTests.java | 2 +- ...essAttributesHealthSerializationTests.java | 2 +- .../ClusterAwarenessHealthTests.java | 2 +- .../cluster/block/ClusterBlockTests.java | 4 +-- .../CompressedStreamUtilsTests.java | 4 +-- .../CoordinationMetadataTests.java | 4 +-- .../coordination/CoordinatorTests.java | 2 +- .../coordination/FollowersCheckerTests.java | 2 +- .../cluster/coordination/JoinHelperTests.java | 4 +-- .../coordination/LeaderCheckerTests.java | 2 +- .../OpenSearchNodeCommandTests.java | 4 +-- .../coordination/PreVoteCollectorTests.java | 2 +- .../PublicationTransportHandlerTests.java | 2 +- .../health/ClusterIndexHealthTests.java | 2 +- .../health/ClusterShardHealthTests.java | 2 +- .../health/ClusterStateHealthTests.java | 2 +- .../cluster/metadata/AliasMetadataTests.java | 2 +- .../ComponentTemplateMetadataTests.java | 2 +- .../metadata/ComponentTemplateTests.java | 2 +- .../ComposableIndexTemplateMetadataTests.java | 2 +- .../ComposableIndexTemplateTests.java | 2 +- .../metadata/DataStreamMetadataTests.java | 2 +- .../metadata/DataStreamTemplateTests.java | 2 +- .../cluster/metadata/DataStreamTests.java | 4 +-- ...onAttributeMetadataSerializationTests.java | 4 +-- .../DecommissionAttributeMetadataTests.java | 2 +- .../cluster/metadata/IndexGraveyardTests.java | 4 +-- .../cluster/metadata/IndexMetadataTests.java | 10 +++---- .../IndexNameExpressionResolverTests.java | 2 +- .../metadata/IndexTemplateMetadataTests.java | 4 +-- .../cluster/metadata/ManifestTests.java | 4 +-- .../MetadataCreateIndexServiceTests.java | 2 +- .../MetadataDeleteIndexServiceTests.java | 2 +- .../MetadataIndexAliasesServiceTests.java | 2 +- .../MetadataIndexStateServiceTests.java | 4 +-- .../MetadataIndexStateServiceUtils.java | 2 +- .../MetadataIndexTemplateServiceTests.java | 2 +- .../metadata/MetadataMappingServiceTests.java | 2 +- .../cluster/metadata/MetadataTests.java | 8 +++--- .../metadata/TemplateUpgradeServiceTests.java | 4 +-- .../cluster/node/DiscoveryNodeTests.java | 2 +- .../cluster/routing/AllocationIdTests.java | 4 +-- .../FailAwareWeightedRoutingTests.java | 2 +- .../routing/GroupShardsIteratorTests.java | 4 +-- .../routing/IndexShardRoutingTableTests.java | 4 +-- .../routing/OperationRoutingTests.java | 4 +-- .../routing/PlainShardIteratorTests.java | 2 +- .../cluster/routing/RoutingNodeTests.java | 4 +-- .../routing/RoutingTableGenerator.java | 2 +- .../cluster/routing/RoutingTableTests.java | 4 +-- .../cluster/routing/ShardRoutingTests.java | 4 +-- .../cluster/routing/UnassignedInfoTests.java | 4 +-- .../allocation/AllocationCommandsTests.java | 10 +++---- .../allocation/BalanceConfigurationTests.java | 2 +- .../allocation/DiskThresholdMonitorTests.java | 2 +- .../allocation/FailedShardsRoutingTests.java | 2 +- ...dexShardConstraintDeciderOverlapTests.java | 2 +- .../NodeVersionAllocationDeciderTests.java | 4 +-- .../ResizeAllocationDeciderTests.java | 4 +-- .../allocation/SameShardRoutingTests.java | 2 +- .../allocation/StartedShardsRoutingTests.java | 4 +-- .../allocation/ThrottlingAllocationTests.java | 4 +-- .../decider/AllocationDecidersTests.java | 2 +- .../decider/DiskThresholdDeciderTests.java | 4 +-- .../DiskThresholdDeciderUnitTests.java | 4 +-- ...storeInProgressAllocationDeciderTests.java | 2 +- .../ClusterSerializationTests.java | 8 +++--- .../cluster/serialization/DiffableTests.java | 4 +-- .../ClusterManagerThrottlingStatsTests.java | 2 +- .../structure/RoutingIteratorTests.java | 2 +- .../org/opensearch/common/ChannelsTests.java | 4 +-- .../common/FieldMemoryStatsTests.java | 2 +- .../opensearch/common/RoundingWireTests.java | 2 +- .../bytes/ByteBuffersBytesReferenceTests.java | 2 ++ .../common/bytes/BytesArrayTests.java | 2 ++ .../bytes/CompositeBytesReferenceTests.java | 3 ++ .../bytes/PagedBytesReferenceTests.java | 5 +++- .../bytes/ReleasableBytesReferenceTests.java | 5 +++- .../DeflateCompressedXContentTests.java | 2 +- .../common/geo/GeoDistanceTests.java | 2 +- .../common/geo/GeoJsonSerializationTests.java | 4 +-- .../opensearch/common/geo/GeoUtilTests.java | 2 +- .../common/geo/GeometryIOTests.java | 6 ++-- .../common/geo/ShapeRelationTests.java | 2 +- .../common/geo/SpatialStrategyTests.java | 2 +- .../AbstractShapeBuilderTestCase.java | 6 ++-- .../io/VersionedCodecStreamWrapperTests.java | 2 +- .../stream/AbstractWriteableEnumTestCase.java | 3 ++ .../common/io/stream/BytesStreamsTests.java | 10 +++++-- .../io/stream/DelayableWriteableTests.java | 5 ++++ .../opensearch/common/lucene/LuceneTests.java | 2 +- .../common/lucene/ShardCoreKeyMapTests.java | 2 +- .../index/OpenSearchDirectoryReaderTests.java | 2 +- .../common/lucene/uid/VersionsTests.java | 2 +- .../common/network/NetworkModuleTests.java | 2 +- .../common/settings/SecureStringTests.java | 1 + .../common/settings/SettingTests.java | 4 +-- .../common/settings/SettingsTests.java | 5 ++-- .../settings/WriteableSettingTests.java | 4 +-- .../transport/BoundTransportAddressTests.java | 2 +- .../common/unit/ByteSizeUnitTests.java | 2 +- .../common/unit/ByteSizeValueTests.java | 2 +- .../common/unit/DistanceUnitTests.java | 2 +- .../common/unit/FuzzinessTests.java | 2 +- .../common/util/BigArraysTests.java | 2 ++ .../common/util/CuckooFilterTests.java | 2 +- .../SetBackedScalingCuckooFilterTests.java | 2 +- .../common/xcontent/BaseXContentTestCase.java | 6 ++-- .../common/xcontent/XContentFactoryTests.java | 4 +-- .../xcontent/XContentParserUtilsTests.java | 13 +++++---- .../builder/XContentBuilderTests.java | 4 +-- .../cbor/CborXContentParserTests.java | 2 +- .../support/AbstractFilteringTestCase.java | 2 +- .../xcontent/support/XContentHelperTests.java | 2 +- .../support/XContentMapValuesTests.java | 2 +- .../AbstractXContentFilteringTestCase.java | 2 +- .../discovery/DiscoveryModuleTests.java | 2 +- .../FileBasedSeedHostsProviderTests.java | 2 +- .../InitializeExtensionRequestTests.java | 4 +-- .../InitializeExtensionResponseTests.java | 4 +-- .../opensearch/discovery/PeerFinderTests.java | 2 +- .../discovery/SeedHostsResolverTests.java | 2 +- .../opensearch/env/NodeEnvironmentTests.java | 4 +-- .../env/NodeRepurposeCommandTests.java | 2 +- .../extensions/ExtensionResponseTests.java | 4 +-- .../extensions/ExtensionsManagerTests.java | 6 ++-- .../action/ExtensionActionRequestTests.java | 4 +-- .../action/ExtensionActionResponseTests.java | 4 +-- .../action/ExtensionActionUtilTests.java | 8 +++--- .../ExtensionHandleTransportRequestTests.java | 4 +-- ...ExtensionTransportActionsHandlerTests.java | 2 +- .../RegisterTransportActionsRequestTests.java | 2 +- .../RemoteExtensionActionResponseTests.java | 4 +-- ...nsportActionRequestFromExtensionTests.java | 4 +-- .../rest/ExtensionRestRequestTests.java | 12 ++++---- .../rest/ExtensionRestResponseTests.java | 11 ++++---- .../rest/RegisterRestActionsTests.java | 4 +-- .../RestInitializeExtensionActionTests.java | 6 ++-- .../rest/RestSendToExtensionActionTests.java | 2 +- .../settings/RegisterCustomSettingsTests.java | 4 +-- .../gateway/AsyncShardFetchTests.java | 2 +- .../gateway/ClusterStateUpdatersTests.java | 2 +- .../gateway/DanglingIndicesStateTests.java | 2 +- .../IncrementalClusterStateWriterTests.java | 2 +- .../gateway/MetaStateServiceTests.java | 2 +- .../PersistedClusterStateServiceTests.java | 2 +- .../gateway/PrimaryShardAllocatorTests.java | 2 +- .../gateway/PriorityComparatorTests.java | 2 +- .../gateway/ReplicaShardAllocatorTests.java | 2 +- .../AbstractHttpServerTransportTests.java | 6 ++-- .../org/opensearch/http/CorsHandlerTests.java | 4 +-- .../http/DefaultRestChannelTests.java | 8 +++--- .../org/opensearch/http/TestHttpRequest.java | 6 ++-- .../org/opensearch/http/TestHttpResponse.java | 4 +-- .../opensearch/index/IndexModuleTests.java | 3 +- .../opensearch/index/IndexServiceTests.java | 1 + .../java/org/opensearch/index/IndexTests.java | 3 +- .../index/IndexingPressureServiceTests.java | 3 +- .../index/IndexingSlowLogTests.java | 7 +++-- .../index/MergePolicySettingsTests.java | 2 +- .../opensearch/index/SearchSlowLogTests.java | 2 +- ...egmentReplicationPressureServiceTests.java | 2 +- ...exingPressureConcurrentExecutionTests.java | 3 +- ...ardIndexingPressureMemoryManagerTests.java | 3 +- .../ShardIndexingPressureStoreTests.java | 2 +- .../index/ShardIndexingPressureTests.java | 3 +- .../ShardIndexingPressureTrackerTests.java | 2 +- .../ReloadableCustomAnalyzerTests.java | 2 +- .../cache/bitset/BitSetFilterCacheTests.java | 2 +- .../index/engine/InternalEngineTests.java | 6 ++-- .../index/engine/ReadOnlyEngineTests.java | 2 +- .../opensearch/index/engine/SegmentTests.java | 2 +- .../fielddata/AbstractFieldDataTestCase.java | 2 +- .../AbstractStringFieldDataTestCase.java | 2 +- .../fielddata/BinaryDVFieldDataTests.java | 2 +- .../index/fielddata/FieldDataCacheTests.java | 2 +- .../index/fielddata/FieldDataStatsTests.java | 2 +- .../fielddata/IndexFieldDataServiceTests.java | 2 +- .../FieldStatsProviderRefreshTests.java | 2 +- .../index/get/DocumentFieldTests.java | 2 +- .../opensearch/index/get/GetResultTests.java | 6 ++-- .../index/mapper/BinaryFieldMapperTests.java | 4 +-- .../mapper/CompletionFieldMapperTests.java | 2 +- .../index/mapper/CopyToMapperTests.java | 2 +- .../mapper/DataStreamFieldMapperTests.java | 2 +- .../index/mapper/DocumentParserTests.java | 4 +-- .../index/mapper/DynamicMappingTests.java | 2 +- .../mapper/FieldNamesFieldMapperTests.java | 2 +- .../mapper/FlatObjectFieldDataTests.java | 2 +- .../GenericStoreDynamicTemplateTests.java | 2 +- .../index/mapper/IdFieldMapperTests.java | 4 +-- .../index/mapper/IndexFieldMapperTests.java | 2 +- .../index/mapper/IpRangeFieldMapperTests.java | 2 +- .../mapper/JavaMultiFieldMergeTests.java | 2 +- .../index/mapper/MapperServiceTests.java | 2 +- .../index/mapper/MultiFieldTests.java | 4 +-- .../index/mapper/NestedObjectMapperTests.java | 2 +- .../mapper/NestedPathFieldMapperTests.java | 2 +- .../mapper/NullValueObjectMappingTests.java | 2 +- .../index/mapper/NumberFieldMapperTests.java | 2 +- .../index/mapper/ObjectMapperTests.java | 2 +- .../mapper/PathMatchDynamicTemplateTests.java | 2 +- .../index/mapper/RoutingFieldMapperTests.java | 2 +- .../index/mapper/SourceFieldMapperTests.java | 4 +-- .../mapper/StoredNumericValuesTests.java | 2 +- .../index/mapper/TypeParsersTests.java | 2 +- .../index/mapper/UpdateMappingTests.java | 2 +- .../query/AbstractQueryBuilderTests.java | 2 +- .../index/query/CombineFunctionTests.java | 2 +- .../CombineIntervalsSourceProviderTests.java | 4 +-- .../query/CommonTermsQueryBuilderTests.java | 2 +- .../query/ConstantScoreQueryBuilderTests.java | 2 +- ...sjunctionIntervalsSourceProviderTests.java | 4 +-- .../FieldMaskingSpanQueryBuilderTests.java | 2 +- .../FilterIntervalsSourceProviderTests.java | 4 +-- .../FuzzyIntervalsSourceProviderTests.java | 2 +- .../index/query/FuzzyQueryBuilderTests.java | 2 +- .../query/GeoDistanceQueryBuilderTests.java | 2 +- .../query/GeoPolygonQueryBuilderTests.java | 2 +- .../query/GeoShapeQueryBuilderTests.java | 2 +- .../index/query/IdsQueryBuilderTests.java | 2 +- .../index/query/InnerHitBuilderTests.java | 2 +- .../query/IntervalQueryBuilderTests.java | 2 +- .../MatchBoolPrefixQueryBuilderTests.java | 2 +- .../MatchIntervalsSourceProviderTests.java | 4 +-- .../MatchPhrasePrefixQueryBuilderTests.java | 2 +- .../query/MatchPhraseQueryBuilderTests.java | 2 +- .../index/query/MatchQueryBuilderTests.java | 2 +- .../query/MoreLikeThisQueryBuilderTests.java | 2 +- .../query/MultiMatchQueryBuilderTests.java | 2 +- .../opensearch/index/query/OperatorTests.java | 2 +- .../PrefixIntervalsSourceProviderTests.java | 2 +- .../index/query/PrefixQueryBuilderTests.java | 2 +- .../index/query/QueryShardContextTests.java | 4 +-- .../index/query/QueryShardExceptionTests.java | 2 +- .../index/query/RangeQueryBuilderTests.java | 2 +- .../RegexpIntervalsSourceProviderTests.java | 2 +- .../index/query/RegexpQueryBuilderTests.java | 2 +- .../index/query/ScoreModeTests.java | 2 +- .../index/query/ScriptQueryBuilderTests.java | 2 +- .../SpanContainingQueryBuilderTests.java | 2 +- .../query/SpanFirstQueryBuilderTests.java | 2 +- .../query/SpanMultiTermQueryBuilderTests.java | 2 +- .../query/SpanNearQueryBuilderTests.java | 2 +- .../index/query/SpanNotQueryBuilderTests.java | 2 +- .../index/query/SpanOrQueryBuilderTests.java | 2 +- .../query/SpanTermQueryBuilderTests.java | 2 +- .../query/SpanWithinQueryBuilderTests.java | 2 +- .../index/query/TermQueryBuilderTests.java | 2 +- .../index/query/TermsQueryBuilderTests.java | 4 +-- .../WildcardIntervalsSourceProviderTests.java | 2 +- .../query/WildcardQueryBuilderTests.java | 2 +- .../index/query/WrapperQueryBuilderTests.java | 6 ++-- ...FieldValueFactorFunctionModifierTests.java | 2 +- .../FunctionScoreQueryBuilderTests.java | 6 ++-- .../index/query/plugin/DummyQueryBuilder.java | 4 +-- .../index/refresh/RefreshStatsTests.java | 2 +- .../reindex/BulkByScrollResponseTests.java | 2 +- .../index/reindex/ReindexRequestTests.java | 6 ++-- ...oteRefreshSegmentPressureServiceTests.java | 2 +- .../RemoteRefreshSegmentTrackerTests.java | 4 +-- .../RecoveryDuringReplicationTests.java | 2 +- .../RetentionLeasesReplicationTests.java | 2 +- .../search/geo/GeoPointParsingTests.java | 2 +- .../AbstractNumberNestedSortingTestCase.java | 2 +- .../search/nested/NestedSortingTests.java | 2 +- .../GlobalCheckpointSyncActionTests.java | 4 +-- ...PeerRecoveryRetentionLeaseExpiryTests.java | 2 +- ...ReplicationTrackerRetentionLeaseTests.java | 2 +- .../seqno/ReplicationTrackerTestCase.java | 2 +- .../index/seqno/ReplicationTrackerTests.java | 4 +-- .../seqno/RetentionLeaseActionsTests.java | 2 +- ...tentionLeaseBackgroundSyncActionTests.java | 4 +-- .../index/seqno/RetentionLeaseStatsTests.java | 2 +- ...tentionLeaseStatsWireSerializingTests.java | 2 +- .../seqno/RetentionLeaseSyncActionTests.java | 4 +-- .../index/seqno/RetentionLeaseTests.java | 2 +- .../index/shard/DocsStatsTests.java | 4 +-- .../shard/GlobalCheckpointListenersTests.java | 3 +- .../index/shard/IndexReaderWrapperTests.java | 1 + .../IndexShardOperationPermitsTests.java | 1 + .../index/shard/IndexShardTests.java | 5 ++-- .../shard/IndexingOperationListenerTests.java | 3 +- .../index/shard/NewPathForShardTests.java | 1 + .../shard/PrimaryReplicaSyncerTests.java | 10 +++---- .../index/shard/RefreshListenersTests.java | 7 +++-- .../RemoveCorruptedShardDataCommandTests.java | 1 + .../index/shard/ShardGetServiceTests.java | 1 + .../opensearch/index/shard/ShardIdTests.java | 3 +- .../index/shard/ShardPathTests.java | 3 +- .../index/shard/ShardUtilsTests.java | 1 + .../index/shard/StoreRecoveryTests.java | 1 + .../snapshots/blobstore/FileInfoTests.java | 2 +- ...oteStoreShardShallowCopySnapshotTests.java | 2 +- .../index/store/FsDirectoryFactoryTests.java | 4 +-- .../store/RemoteBufferedIndexOutputTests.java | 4 +-- ...moteSegmentStoreDirectoryFactoryTests.java | 2 +- .../RemoteSegmentStoreDirectoryTests.java | 2 +- .../opensearch/index/store/StoreTests.java | 8 +++--- .../opensearch/index/store/StoreUtils.java | 2 +- .../filecache/FileCacheCleanerTests.java | 2 +- .../remote/filecache/FileCacheStatsTests.java | 2 +- .../RemoteSegmentMetadataHandlerTests.java | 2 +- .../suggest/stats/CompletionsStatsTests.java | 2 +- .../index/translog/LocalTranslogTests.java | 8 +++--- .../index/translog/RemoteFSTranslogTests.java | 4 +-- .../index/translog/TestTranslog.java | 2 +- .../translog/TranslogDeletionPolicyTests.java | 2 +- .../index/translog/TranslogHeaderTests.java | 2 +- .../translog/TranslogManagerTestCase.java | 8 +++--- .../listener/TranslogListenerTests.java | 4 +-- .../transfer/FileTransferTrackerTests.java | 2 +- .../TranslogTransferManagerTests.java | 4 +-- .../TranslogTransferMetadataHandlerTests.java | 2 +- ...dicesLifecycleListenerSingleNodeTests.java | 4 +-- .../indices/IndicesQueryCacheTests.java | 2 +- .../indices/IndicesRequestCacheTests.java | 6 ++-- .../indices/IndicesServiceCloseTests.java | 4 +-- .../indices/IndicesServiceTests.java | 4 +-- .../indices/ShardLimitValidatorTests.java | 2 +- .../opensearch/indices/TermsLookupTests.java | 2 +- ...actIndicesClusterStateServiceTestCase.java | 4 +-- .../indices/cluster/ClusterStateChanges.java | 2 +- ...ClusterStateServiceRandomUpdatesTests.java | 4 +-- ...alStorePeerRecoverySourceHandlerTests.java | 6 ++-- .../PeerRecoveryTargetServiceTests.java | 2 +- .../indices/recovery/RecoveryTargetTests.java | 6 ++-- .../indices/recovery/RecoveryTests.java | 2 +- .../recovery/StartRecoveryRequestTests.java | 6 ++-- .../OngoingSegmentReplicationsTests.java | 2 +- .../SegmentFileTransferHandlerTests.java | 2 +- .../SegmentReplicationSourceServiceTests.java | 4 +-- .../SegmentReplicationTargetServiceTests.java | 2 +- .../SegmentReplicationTargetTests.java | 2 +- .../PublishCheckpointActionTests.java | 4 +-- .../replication/common/CopyStateTests.java | 2 +- .../InternalOrPrivateSettingsPlugin.java | 4 +-- .../indices/store/IndicesStoreTests.java | 2 +- .../ingest/IngestMetadataTests.java | 4 +-- .../opensearch/ingest/IngestServiceTests.java | 2 +- .../opensearch/ingest/IngestStatsTests.java | 2 +- .../ingest/PipelineConfigurationTests.java | 6 ++-- .../opensearch/monitor/os/OsStatsTests.java | 2 +- .../node/InternalSettingsPreparerTests.java | 2 +- .../nodesinfo/NodeInfoStreamingTests.java | 2 +- .../CancelPersistentTaskRequestTests.java | 2 +- .../PersistentTasksCustomMetadataTests.java | 14 +++++----- .../PersistentTasksExecutorResponseTests.java | 4 +-- ...PersistentTasksNodeServiceStatusTests.java | 2 +- .../RestartPersistentTaskRequestTests.java | 2 +- .../StartPersistentActionRequestTests.java | 6 ++-- .../persistent/TestPersistentTasksPlugin.java | 8 +++--- .../UpdatePersistentTaskRequestTests.java | 4 +-- .../opensearch/plugins/PluginInfoTests.java | 2 +- .../recovery/ReplicationCollectionTests.java | 2 +- .../opensearch/repositories/IndexIdTests.java | 2 +- .../RepositoriesServiceTests.java | 2 +- .../repositories/RepositoryDataTests.java | 2 +- .../BlobStoreRepositoryRestoreTests.java | 4 +-- .../blobstore/BlobStoreRepositoryTests.java | 2 +- .../repositories/fs/FsRepositoryTests.java | 2 +- .../opensearch/rest/BaseRestHandlerTests.java | 2 +- .../rest/BytesRestResponseTests.java | 7 +++-- .../opensearch/rest/RestControllerTests.java | 5 ++-- .../rest/RestHttpResponseHeadersTests.java | 3 +- .../org/opensearch/rest/RestRequestTests.java | 4 +-- .../rest/action/RestActionListenerTests.java | 2 +- .../rest/action/RestActionsTests.java | 8 +++--- .../rest/action/RestBuilderListenerTests.java | 4 +-- .../rest/action/RestMainActionTests.java | 4 +-- ...tClusterAddWeightedRoutingActionTests.java | 2 +- ...usterDeleteWeightedRoutingActionTests.java | 2 +- .../admin/indices/RestAnalyzeActionTests.java | 2 +- .../indices/RestCreateIndexActionTests.java | 2 +- .../indices/RestGetAliasesActionTests.java | 4 +-- .../indices/RestValidateQueryActionTests.java | 4 +-- .../RestCatSegmentReplicationActionTests.java | 6 ++-- .../action/cat/RestIndicesActionTests.java | 4 +-- .../action/cat/RestRecoveryActionTests.java | 6 ++-- .../action/document/RestBulkActionTests.java | 2 +- .../document/RestGetSourceActionTests.java | 6 ++-- .../action/document/RestIndexActionTests.java | 2 +- .../document/RestUpdateActionTests.java | 2 +- .../opensearch/script/ScriptCacheTests.java | 2 +- .../script/ScriptContextInfoTests.java | 2 +- .../script/ScriptExceptionTests.java | 8 +++--- .../script/ScriptMetadataTests.java | 6 ++-- .../opensearch/script/ScriptServiceTests.java | 4 +-- .../org/opensearch/script/ScriptTests.java | 4 +-- .../script/StoredScriptSourceTests.java | 4 +-- .../opensearch/script/StoredScriptTests.java | 6 ++-- .../search/AbstractSearchTestCase.java | 8 +++--- .../search/ClearScrollResponseTests.java | 2 +- .../search/DefaultSearchContextTests.java | 2 +- .../search/DeletePitResponseTests.java | 2 +- .../search/DocValueFormatTests.java | 8 +++--- .../search/MultiValueModeTests.java | 2 +- .../search/NestedIdentityTests.java | 2 +- .../org/opensearch/search/SearchHitTests.java | 10 +++---- .../opensearch/search/SearchHitsTests.java | 8 +++--- .../opensearch/search/SearchModuleTests.java | 4 +-- .../opensearch/search/SearchServiceTests.java | 10 +++---- .../SearchSortValuesAndFormatsTests.java | 4 +-- .../search/SearchSortValuesTests.java | 2 +- .../aggregations/AggregationsTests.java | 4 +-- .../AggregatorFactoriesBuilderTests.java | 4 +-- .../AggregatorFactoriesTests.java | 6 ++-- .../InternalAggregationsTests.java | 6 ++-- .../aggregations/InternalOrderTests.java | 2 +- .../SubAggCollectionModeTests.java | 2 +- .../histogram/DateIntervalWrapperTests.java | 2 +- .../bucket/histogram/DoubleBoundsTests.java | 4 +-- .../bucket/histogram/LongBoundsTests.java | 4 +-- .../range/RangeAggregationBuilderTests.java | 2 +- .../terms/RareTermsAggregatorTests.java | 2 +- .../terms/SignificanceHeuristicTests.java | 10 +++---- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../metrics/InternalTopHitsTests.java | 2 +- .../metrics/PercentilesMethodTests.java | 2 +- .../metrics/TopHitsAggregatorTests.java | 2 +- .../WeightedAvgAggregationBuilderTests.java | 2 +- .../pipeline/BucketHelpersTests.java | 2 +- .../aggregations/pipeline/GapPolicyTests.java | 2 +- .../MultiTermsValuesSourceConfigTests.java | 4 +-- .../MultiValuesSourceFieldConfigTests.java | 4 +-- .../SearchBackpressureServiceTests.java | 4 +-- .../stats/SearchBackpressureStatsTests.java | 2 +- .../stats/SearchShardTaskStatsTests.java | 2 +- .../stats/SearchTaskStatsTests.java | 2 +- .../builder/SearchSourceBuilderTests.java | 8 +++--- .../search/collapse/CollapseBuilderTests.java | 4 +-- .../fetch/subphase/FetchSourcePhaseTests.java | 2 +- .../fetch/subphase/FieldFetcherTests.java | 2 +- .../subphase/highlight/CustomHighlighter.java | 2 +- .../highlight/HighlightBuilderTests.java | 8 +++--- .../highlight/HighlightFieldTests.java | 4 +-- .../internal/ContextIndexSearcherTests.java | 2 +- .../internal/ShardSearchRequestTests.java | 2 +- .../pipeline/SearchPipelineInfoTests.java | 2 +- .../pipeline/SearchPipelineServiceTests.java | 4 +-- .../pipeline/SearchPipelineStatsTests.java | 4 +-- .../search/pit/RestDeletePitActionTests.java | 2 +- .../search/profile/ProfileResultTests.java | 4 +-- .../SearchProfileShardResultsTests.java | 6 ++-- .../AggregationProfileShardResultTests.java | 4 +-- .../profile/query/CollectorResultTests.java | 4 +-- .../query/QueryProfileShardResultTests.java | 4 +-- .../search/query/QuerySearchResultTests.java | 4 +-- .../rescore/QueryRescorerBuilderTests.java | 4 +-- .../scroll/RestClearScrollActionTests.java | 2 +- .../scroll/RestSearchScrollActionTests.java | 2 +- .../searchafter/SearchAfterBuilderTests.java | 6 ++-- .../search/slice/SliceBuilderTests.java | 4 +-- .../search/sort/AbstractSortTestCase.java | 4 +-- .../search/sort/NestedSortBuilderTests.java | 2 +- .../search/sort/SortOrderTests.java | 2 +- .../search/sort/SortValueTests.java | 2 +- .../search/sort/plugin/CustomSortBuilder.java | 4 +-- .../AbstractSuggestionBuilderTestCase.java | 6 ++-- .../CompletionSuggestionOptionTests.java | 4 +-- .../search/suggest/SuggestBuilderTests.java | 2 +- .../search/suggest/SuggestTests.java | 14 +++++----- .../search/suggest/SuggestionEntryTests.java | 6 ++-- .../search/suggest/SuggestionOptionTests.java | 6 ++-- .../search/suggest/SuggestionTests.java | 6 ++-- .../suggest/TermSuggestionOptionTests.java | 6 ++-- .../CategoryContextMappingTests.java | 2 +- .../CompletionSuggesterBuilderTests.java | 2 +- .../completion/CompletionSuggestionTests.java | 2 +- .../suggest/completion/FuzzyOptionsTests.java | 2 +- .../completion/GeoContextMappingTests.java | 2 +- .../completion/GeoQueryContextTests.java | 2 +- .../suggest/completion/RegexOptionsTests.java | 2 +- .../phrase/DirectCandidateGeneratorTests.java | 2 +- .../phrase/SmoothingModelTestCase.java | 2 +- .../snapshots/BlobStoreFormatTests.java | 4 +-- .../InternalSnapshotsInfoServiceTests.java | 4 +-- ...epositoriesMetadataSerializationTests.java | 4 +-- .../snapshots/RestoreServiceTests.java | 2 +- .../snapshots/SnapshotInfoTests.java | 4 +-- .../snapshots/SnapshotRequestsTests.java | 2 +- .../snapshots/SnapshotResiliencyTests.java | 4 +-- .../snapshots/SnapshotShardsServiceTests.java | 2 +- .../snapshots/SnapshotUtilsTests.java | 2 +- ...SnapshotsInProgressSerializationTests.java | 10 +++---- .../snapshots/SnapshotsServiceTests.java | 4 +-- .../MockEventuallyConsistentRepository.java | 2 +- ...SearchShardTaskCancellationStatsTests.java | 2 +- ...askCancellationMonitoringServiceTests.java | 2 +- .../tasks/TaskCancellationStatsTests.java | 2 +- .../org/opensearch/tasks/TaskIdTests.java | 4 +-- .../org/opensearch/tasks/TaskInfoTests.java | 6 ++-- .../org/opensearch/tasks/TaskResultTests.java | 6 ++-- .../ThreadPoolSerializationTests.java | 4 +-- .../CompressibleBytesOutputStreamTests.java | 8 +++--- .../transport/InboundAggregatorTests.java | 2 +- .../transport/InboundDecoderTests.java | 2 +- .../transport/InboundHandlerTests.java | 10 +++---- .../transport/InboundPipelineTests.java | 4 +-- .../transport/OutboundHandlerTests.java | 4 +-- .../RemoteClusterConnectionTests.java | 2 +- .../transport/TransportActionProxyTests.java | 4 +-- .../transport/TransportDecompressorTests.java | 10 +++---- .../transport/TransportHandshakerTests.java | 2 +- .../transport/TransportKeepAliveTests.java | 2 +- .../transport/TransportLoggerTests.java | 2 +- ...ortServiceDeserializationFailureTests.java | 2 +- .../TransportServiceHandshakeTests.java | 2 +- .../DelayedShardAggregationBuilder.java | 4 +-- .../java/fixture/azure/AzureHttpHandler.java | 6 ++-- .../fixture/gcs/FakeOAuth2HttpHandler.java | 2 +- .../gcs/GoogleCloudStorageHttpHandler.java | 8 +++--- .../main/java/fixture/s3/S3HttpFixture.java | 2 +- .../java/fixture/s3/S3HttpFixtureWithEC2.java | 2 +- .../java/fixture/s3/S3HttpFixtureWithECS.java | 2 +- .../java/fixture/s3/S3HttpFixtureWithEKS.java | 2 +- .../s3/S3HttpFixtureWithSessionToken.java | 2 +- .../main/java/fixture/s3/S3HttpHandler.java | 6 ++-- .../ClusterStateCreationUtils.java | 2 +- .../cluster/DataStreamTestHelper.java | 2 +- .../AbstractCoordinatorTestCase.java | 6 ++-- .../cluster/routing/TestShardRouting.java | 2 +- .../bytes/AbstractBytesReferenceTestCase.java | 7 +++-- .../common/settings/MockSecureSettings.java | 1 + .../opensearch/common/util/MockBigArrays.java | 2 ++ .../index/engine/EngineTestCase.java | 8 +++--- .../index/mapper/MapperServiceTestCase.java | 4 +-- .../index/mapper/MapperTestCase.java | 2 +- ...enSearchIndexLevelReplicationTestCase.java | 6 ++-- .../index/shard/IndexShardTestCase.java | 5 ++-- .../index/shard/RestoreOnlyRepository.java | 1 + .../indices/recovery/AsyncRecoveryTarget.java | 4 +-- .../java/org/opensearch/node/MockNode.java | 2 +- .../AbstractBlobContainerRetriesTestCase.java | 2 +- ...earchBlobStoreRepositoryIntegTestCase.java | 2 +- ...chMockAPIBasedRepositoryIntegTestCase.java | 2 +- .../search/RandomSearchRequestGenerator.java | 4 +-- .../aggregations/AggregatorTestCase.java | 6 ++-- .../aggregations/BaseAggregationTestCase.java | 4 +-- .../BasePipelineAggregationTestCase.java | 6 ++-- ...ternalSingleBucketAggregationTestCase.java | 2 +- .../BaseCompositeAggregatorTestCase.java | 4 +-- .../AbstractSnapshotIntegTestCase.java | 2 +- .../AbstractBroadcastResponseTestCase.java | 10 +++---- .../test/AbstractBuilderTestCase.java | 6 ++-- ...AbstractDiffableSerializationTestCase.java | 2 +- ...ractDiffableWireSerializationTestCase.java | 2 +- .../test/AbstractNamedWriteableTestCase.java | 6 ++-- .../test/AbstractQueryTestCase.java | 10 +++---- .../test/AbstractSerializingTestCase.java | 2 +- .../test/AbstractWireSerializingTestCase.java | 8 +++--- .../opensearch/test/AbstractWireTestCase.java | 6 ++-- .../test/AbstractXContentTestCase.java | 2 +- .../opensearch/test/DiffableTestUtils.java | 10 +++---- .../org/opensearch/test/DummyShardLock.java | 2 +- .../opensearch/test/ExternalTestCluster.java | 2 +- .../opensearch/test/IndexSettingsModule.java | 2 +- .../test/InternalAggregationTestCase.java | 6 ++-- .../opensearch/test/InternalTestCluster.java | 6 ++-- .../test/MockIndexEventListener.java | 4 +-- .../test/OpenSearchIntegTestCase.java | 10 +++---- .../test/OpenSearchSingleNodeTestCase.java | 2 +- .../opensearch/test/OpenSearchTestCase.java | 16 +++++------ .../org/opensearch/test/RandomObjects.java | 8 +++--- .../org/opensearch/test/StreamsUtils.java | 2 +- .../java/org/opensearch/test/TestCluster.java | 2 +- .../opensearch/test/TestCustomMetadata.java | 4 +-- .../opensearch/test/TestSearchContext.java | 2 +- .../opensearch/test/XContentTestUtils.java | 2 +- .../test/client/NoOpNodeClient.java | 2 +- .../test/engine/MockEngineSupport.java | 2 +- .../test/gateway/TestGatewayAllocator.java | 2 +- .../test/hamcrest/OpenSearchAssertions.java | 8 +++--- .../opensearch/test/rest/FakeRestChannel.java | 2 +- .../opensearch/test/rest/FakeRestRequest.java | 6 ++-- .../test/rest/OpenSearchRestTestCase.java | 2 +- .../yaml/ClientYamlTestExecutionContext.java | 2 +- .../rest/yaml/ClientYamlTestResponse.java | 2 +- .../opensearch/test/rest/yaml/ObjectPath.java | 4 +-- .../ClientYamlSuiteRestApiParser.java | 2 +- .../yaml/section/ClientYamlTestSection.java | 2 +- .../yaml/section/ClientYamlTestSuite.java | 2 +- .../test/rest/yaml/section/DoSection.java | 2 +- .../test/rest/yaml/section/SetSection.java | 2 +- .../test/rest/yaml/section/SkipSection.java | 2 +- .../rest/yaml/section/TeardownSection.java | 2 +- .../yaml/section/TransformAndSetSection.java | 2 +- .../test/store/MockFSDirectoryFactory.java | 2 +- .../test/store/MockFSIndexStore.java | 2 +- .../test/transport/MockTransport.java | 6 ++-- .../test/transport/MockTransportService.java | 2 +- .../AbstractSimpleTransportTestCase.java | 4 +-- .../opensearch/transport/FakeTcpChannel.java | 2 +- .../org/opensearch/transport/TestRequest.java | 4 +-- .../opensearch/transport/TestResponse.java | 4 +-- .../transport/nio/MockNioTransport.java | 6 ++-- .../transport/nio/MockNioTransportPlugin.java | 2 +- .../test/AbstractXContentTestCaseTests.java | 2 +- .../test/XContentTestUtilsTests.java | 2 +- .../DisruptableMockTransportTests.java | 2 +- .../test/disruption/NetworkDisruptionIT.java | 2 +- .../hamcrest/OpenSearchAssertionsTests.java | 4 +-- .../test/rest/yaml/ObjectPathTests.java | 2 +- ...entYamlSuiteRestApiParserFailingTests.java | 2 +- .../section/ClientYamlTestSectionTests.java | 2 +- .../section/ClientYamlTestSuiteTests.java | 2 +- .../rest/yaml/section/DoSectionTests.java | 2 +- .../rest/yaml/section/SetSectionTests.java | 2 +- .../rest/yaml/section/SkipSectionTests.java | 2 +- .../section/TransformAndSetSectionTests.java | 2 +- .../test/test/OpenSearchTestCaseTests.java | 2 +- .../nio/SimpleMockNioTransportTests.java | 2 +- 2881 files changed, 5839 insertions(+), 5737 deletions(-) delete mode 100644 libs/core/src/main/java/org/opensearch/common/io/package-info.java delete mode 100644 libs/core/src/main/java/org/opensearch/common/package-info.java delete mode 100644 libs/core/src/main/java/org/opensearch/common/xcontent/package-info.java rename libs/core/src/main/java/org/opensearch/{ => core}/action/ShardOperationFailedException.java (95%) rename libs/core/src/main/java/org/opensearch/{ => core}/action/package-info.java (86%) rename libs/core/src/main/java/org/opensearch/{ => core}/action/support/DefaultShardOperationFailedException.java (94%) rename libs/core/src/main/java/org/opensearch/{ => core}/action/support/package-info.java (85%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/ParsingException.java (95%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/bytes/AbstractBytesReference.java (99%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/bytes/BytesArray.java (97%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/bytes/BytesReference.java (97%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/bytes/CompositeBytesReference.java (99%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/bytes/PagedBytesReference.java (98%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/bytes/package-info.java (85%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/compress/NotXContentException.java (96%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/compress/package-info.java (84%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/ByteBufferStreamInput.java (98%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/BytesStream.java (92%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/BytesStreamInput.java (98%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/DataOutputStreamOutput.java (97%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/FilterStreamInput.java (98%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/InputStreamStreamInput.java (98%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/NamedWriteable.java (96%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/NamedWriteableAwareStreamInput.java (98%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/NamedWriteableRegistry.java (97%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/NotSerializableExceptionWrapper.java (97%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/OutputStreamStreamOutput.java (97%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/StreamInput.java (99%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/StreamOutput.java (99%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/VersionedNamedWriteable.java (97%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/Writeable.java (99%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/io/stream/package-info.java (84%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/settings/SecureString.java (99%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/settings/package-info.java (84%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/text/Text.java (96%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/text/package-info.java (85%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/util/BigArray.java (97%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/util/ByteArray.java (98%) rename libs/core/src/main/java/org/opensearch/{ => core}/common/util/package-info.java (85%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/Index.java (95%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/package-info.java (87%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/shard/ShardId.java (94%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/shard/package-info.java (85%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/snapshots/IndexShardSnapshotException.java (91%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/snapshots/IndexShardSnapshotFailedException.java (91%) rename libs/core/src/main/java/org/opensearch/{ => core}/index/snapshots/package-info.java (84%) rename libs/core/src/main/java/org/opensearch/{ => core}/rest/RestStatus.java (99%) rename libs/core/src/main/java/org/opensearch/{ => core}/rest/package-info.java (87%) rename libs/core/src/main/java/org/opensearch/{common => core}/xcontent/XContentParserUtils.java (96%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/action/support/DefaultShardOperationFailedExceptionTests.java (97%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/StreamsTests.java (95%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/BaseStreamTests.java (98%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/ByteBufferStreamInputTests.java (87%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/BytesReferenceStreamInputTests.java (83%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/BytesStreamInputTests.java (86%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/FilterStreamInputTests.java (86%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/InputStreamStreamInputTests.java (86%) rename {server/src/test/java/org/opensearch => libs/core/src/test/java/org/opensearch/core}/common/io/stream/NamedWriteableRegistryTests.java (98%) diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java index b549de555aca1..76851881730a3 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java @@ -42,11 +42,11 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.NoopCircuitBreaker; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchModule; diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java index e49ae187acea7..8f86a0f3afbc6 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java @@ -34,7 +34,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.stream.DelayableWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java index 4b8107a4d198b..6a75837f8a0fd 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java @@ -41,7 +41,7 @@ import org.opensearch.client.Requests; import org.opensearch.client.node.NodeClient; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; @@ -56,7 +56,7 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.POST; import static org.opensearch.rest.RestRequest.Method.PUT; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; public class RestNoopBulkAction extends BaseRestHandler { diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/TransportNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/TransportNoopBulkAction.java index 22dd96a0bdfc5..4df9a0465b1a6 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/TransportNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/TransportNoopBulkAction.java @@ -41,7 +41,7 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.update.UpdateResponse; import org.opensearch.common.inject.Inject; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/search/TransportNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/search/TransportNoopSearchAction.java index 77668a030ee7f..1b35787d226e7 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/search/TransportNoopSearchAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/search/TransportNoopSearchAction.java @@ -39,7 +39,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java index 10cfec9497862..45511f2bc2a47 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java @@ -47,7 +47,7 @@ import org.opensearch.client.indices.GetComponentTemplatesRequest; import org.opensearch.client.indices.GetComponentTemplatesResponse; import org.opensearch.client.indices.PutComponentTemplateRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/GetAliasesResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/GetAliasesResponse.java index 83b00df73bdc4..ee829e2be1f11 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/GetAliasesResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/GetAliasesResponse.java @@ -39,7 +39,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; @@ -48,7 +48,7 @@ import java.util.Map; import java.util.Set; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Response obtained from the get aliases API. diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java index 9b4586ec6bf89..f20a6f627a680 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java @@ -88,7 +88,7 @@ import org.opensearch.client.indices.SimulateIndexTemplateResponse; import org.opensearch.client.indices.rollover.RolloverRequest; import org.opensearch.client.indices.rollover.RolloverResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java index 007761d9ab473..f3fbadf07d304 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java @@ -73,7 +73,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Priority; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CollectionUtils; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index eb704317c7da8..9574d4b17819d 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -99,7 +99,7 @@ import org.opensearch.index.reindex.UpdateByQueryRequest; import org.opensearch.plugins.spi.NamedXContentProvider; import org.opensearch.rest.BytesRestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.mustache.MultiSearchTemplateRequest; import org.opensearch.script.mustache.MultiSearchTemplateResponse; import org.opensearch.script.mustache.SearchTemplateRequest; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/cluster/RemoteInfoResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/cluster/RemoteInfoResponse.java index 4474d54cf9f88..d1fa2eb1dff0d 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/cluster/RemoteInfoResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/cluster/RemoteInfoResponse.java @@ -38,7 +38,7 @@ import java.util.Collections; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A response to _remote/info API request. diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/core/BroadcastResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/core/BroadcastResponse.java index 368a68d9248cc..42011a0da7ab2 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/core/BroadcastResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/core/BroadcastResponse.java @@ -32,7 +32,7 @@ package org.opensearch.client.core; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/core/CountResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/core/CountResponse.java index cc1df7ed815de..10d318afd9da0 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/core/CountResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/core/CountResponse.java @@ -35,14 +35,14 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A response to _count API request. diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/core/TermVectorsRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/core/TermVectorsRequest.java index 2384cef68ade0..c58114299c065 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/core/TermVectorsRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/core/TermVectorsRequest.java @@ -34,7 +34,7 @@ import org.opensearch.client.Validatable; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java index 674b8bbdf1bc2..a9a851474c424 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java @@ -32,13 +32,13 @@ package org.opensearch.client.indices; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import java.util.List; import java.util.Objects; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java index 2206e732aa00e..3ed41684b090e 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java @@ -39,8 +39,8 @@ import org.opensearch.client.TimedRequest; import org.opensearch.client.Validatable; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetFieldMappingsResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetFieldMappingsResponse.java index 091c7f39e41da..1359c68fc1311 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetFieldMappingsResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetFieldMappingsResponse.java @@ -33,7 +33,7 @@ package org.opensearch.client.indices; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; @@ -49,7 +49,7 @@ import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** Response object for {@link GetFieldMappingsRequest} API */ public class GetFieldMappingsResponse { diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java index a8314aa9ffcbf..6ec1c312c9ba9 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java @@ -49,7 +49,7 @@ import java.util.List; import java.util.Map; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A client side response for a get index action. diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetMappingsResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetMappingsResponse.java index 89d8b2ef3e53f..8257eb02e9ac1 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetMappingsResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetMappingsResponse.java @@ -35,7 +35,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.mapper.MapperService; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java index c6d6a29a02af1..fed958955af99 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java @@ -39,8 +39,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java index 3d05c12b7ba1f..a2237efaed25f 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.TimedRequest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/slm/GetSnapshotLifecyclePolicyResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/slm/GetSnapshotLifecyclePolicyResponse.java index e7fd12012a6c8..c1329e2bacd59 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/slm/GetSnapshotLifecyclePolicyResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/slm/GetSnapshotLifecyclePolicyResponse.java @@ -41,7 +41,7 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; public class GetSnapshotLifecyclePolicyResponse implements ToXContentObject { diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/tasks/OpenSearchException.java b/client/rest-high-level/src/main/java/org/opensearch/client/tasks/OpenSearchException.java index b0ad087808b90..8ce6e71cfeeba 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/tasks/OpenSearchException.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/tasks/OpenSearchException.java @@ -40,7 +40,7 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * client side counterpart of server side diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java index 420109edb66db..707f4246009aa 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java @@ -31,7 +31,7 @@ package org.opensearch.client; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java index 3908126df23ac..5f7df0cd5860c 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java @@ -31,7 +31,7 @@ package org.opensearch.client; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java index 93235b03bfe10..49acbe8cd2bc1 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java @@ -43,7 +43,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java index 0744fe4e6db3e..44bd085788203 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java @@ -41,7 +41,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.transport.RemoteTransportException; import java.util.Collections; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java index a955b97b15e34..2ac0eee407b95 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java @@ -66,7 +66,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.indices.recovery.RecoverySettings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.transport.RemoteClusterService; import org.opensearch.transport.SniffConnectionStrategy; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java index 06a6fb6802e3c..eaf8f4f8efff7 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java @@ -58,7 +58,7 @@ import org.opensearch.client.core.TermVectorsRequest; import org.opensearch.client.core.TermVectorsResponse; import org.opensearch.client.indices.GetIndexRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; @@ -68,7 +68,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.get.GetResult; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.fetch.subphase.FetchSourceContext; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/GetAliasesResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/GetAliasesResponseTests.java index c35d6d426744e..96f958a4ad2d8 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/GetAliasesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/GetAliasesResponseTests.java @@ -35,7 +35,7 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.AbstractXContentTestCase; import java.io.IOException; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java index 524a300963b72..8ca5c5fa58742 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java @@ -108,7 +108,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.metadata.Template; import org.opensearch.common.ValidationException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -124,7 +124,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java index 4e67198422b23..84cd2fb8b20dd 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java @@ -42,7 +42,7 @@ import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.ingest.SimulatePipelineResponse; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ingest.PipelineConfiguration; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java index 8aae33307279b..0f377720b7aed 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java @@ -37,7 +37,7 @@ import org.opensearch.action.ingest.PutPipelineRequest; import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; import org.apache.hc.client5.http.classic.methods.HttpDelete; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java index c1e2420d86d20..ee1f217f47ef2 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java @@ -48,7 +48,7 @@ import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.common.Booleans; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java index cbee86c34c360..65888e79683e3 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java @@ -48,7 +48,7 @@ import org.opensearch.index.reindex.DeleteByQueryAction; import org.opensearch.index.reindex.DeleteByQueryRequest; import org.opensearch.index.reindex.ReindexRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.RawTaskStatus; import org.opensearch.tasks.TaskId; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java index ca880d2f3d022..ee6aeeeacf633 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java @@ -65,8 +65,8 @@ import org.opensearch.client.core.TermVectorsRequest; import org.opensearch.client.indices.AnalyzeRequest; import org.opensearch.common.CheckedBiConsumer; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; import org.opensearch.common.logging.DeprecationLogger; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java index 85f329eb6e895..24b0fda6c18d5 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java @@ -46,7 +46,7 @@ import org.opensearch.client.core.MainRequest; import org.opensearch.client.core.MainResponse; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -64,7 +64,7 @@ import org.opensearch.index.rankeval.PrecisionAtK; import org.opensearch.index.rankeval.RecallAtK; import org.opensearch.join.aggregations.ChildrenAggregationBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java index 41d8ce2a2fed9..fbfab016b1ead 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java @@ -53,7 +53,7 @@ import org.opensearch.client.core.CountRequest; import org.opensearch.client.core.CountResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -65,7 +65,7 @@ import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.join.aggregations.Children; import org.opensearch.join.aggregations.ChildrenAggregationBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.script.mustache.MultiSearchTemplateRequest; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java index ea27d8e9605d9..70ba39a04e8db 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java @@ -13,7 +13,7 @@ import org.opensearch.action.search.GetSearchPipelineResponse; import org.opensearch.action.search.PutSearchPipelineRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java index 1d93dae5b2c5b..df99d26a0a530 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java @@ -56,7 +56,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.RestoreInfo; import org.opensearch.snapshots.SnapshotInfo; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/StoredScriptsIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/StoredScriptsIT.java index 1c61e79604a9f..c86de6ae645c1 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/StoredScriptsIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/StoredScriptsIT.java @@ -37,9 +37,9 @@ import org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.script.StoredScriptSource; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/TasksIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/TasksIT.java index d987e786fff76..24edd5f93bdba 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/TasksIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/TasksIT.java @@ -47,7 +47,7 @@ import org.opensearch.client.tasks.TaskSubmissionResponse; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/UpdateByQueryIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/UpdateByQueryIT.java index d73fa2bd0aa80..e5fbb30d29292 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/UpdateByQueryIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/UpdateByQueryIT.java @@ -46,7 +46,7 @@ import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.UpdateByQueryAction; import org.opensearch.index.reindex.UpdateByQueryRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.tasks.RawTaskStatus; import org.opensearch.tasks.TaskId; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/BroadcastResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/BroadcastResponseTests.java index 362f9111ec4cb..101f00ad25eb8 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/BroadcastResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/BroadcastResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.client.core; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.client.AbstractResponseTestCase; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/CountResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/CountResponseTests.java index 8641f98a51b9b..06912e0c6bb1f 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/CountResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/CountResponseTests.java @@ -34,11 +34,11 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.rest.action.RestActions; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java index 548aac783cf47..03d6e6720ffed 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java @@ -33,8 +33,8 @@ package org.opensearch.client.core; import org.opensearch.client.AbstractResponseTestCase; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/tasks/GetTaskResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/tasks/GetTaskResponseTests.java index 01b980a95f1b1..63e016fea1da7 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/tasks/GetTaskResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/tasks/GetTaskResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.client.Requests; import org.opensearch.client.tasks.GetTaskResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.tasks.RawTaskStatus; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java index 38addb91f0d41..a973753aa2032 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java @@ -75,7 +75,7 @@ import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.CreateIndexResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; @@ -94,7 +94,7 @@ import org.opensearch.index.reindex.RemoteInfo; import org.opensearch.index.reindex.ScrollableHitSource; import org.opensearch.index.reindex.UpdateByQueryRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.fetch.subphase.FetchSourceContext; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java index a3803701be718..f85fcae7af365 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java @@ -67,7 +67,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.indices.recovery.RecoverySettings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.HashMap; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java index a1334062064d2..73346bc57646e 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java @@ -61,7 +61,7 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; @@ -113,7 +113,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java index 5654791347832..6c11aff3d292e 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java @@ -48,7 +48,7 @@ import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ingest.PipelineConfiguration; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java index d514f66ff36d9..7f7062f0e8a4c 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java @@ -65,9 +65,9 @@ import org.opensearch.client.core.CountResponse; import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.CreateIndexResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentFactory; @@ -86,7 +86,7 @@ import org.opensearch.index.rankeval.RatedDocument; import org.opensearch.index.rankeval.RatedRequest; import org.opensearch.index.rankeval.RatedSearchHit; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.ScriptType; import org.opensearch.script.mustache.MultiSearchTemplateRequest; import org.opensearch.script.mustache.MultiSearchTemplateResponse; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java index c70f5dbade5d3..976c69910d309 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java @@ -66,7 +66,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.RestoreInfo; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java index bd9cbbdc2f450..742dfa69e718b 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java @@ -42,8 +42,8 @@ import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java index 6e3a0b8cdcfc7..b83bdab899a41 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java @@ -35,16 +35,16 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.client.AbstractResponseTestCase; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.transport.ActionNotFoundTransportException; import java.io.IOException; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java index 62ad3ce44462f..96db2fb43c161 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.admin.indices.datastream.DataStreamsStatsAction; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.client.AbstractResponseTestCase; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentParser; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetDataStreamResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetDataStreamResponseTests.java index 194b4f77ff8d8..523758c11de6b 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetDataStreamResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetDataStreamResponseTests.java @@ -40,7 +40,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.ArrayList; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetFieldMappingsResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetFieldMappingsResponseTests.java index 718abe31f6fb9..115685a2c5afc 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetFieldMappingsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetFieldMappingsResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.client.indices; import org.opensearch.client.indices.GetFieldMappingsResponse.FieldMappingMetadata; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java index 79d4683ae09ae..6f924898096a0 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java @@ -34,8 +34,8 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java index 2a39b0deb8827..e6411b615df07 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java @@ -38,8 +38,8 @@ import org.opensearch.client.AbstractResponseTestCase; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.transport.TransportAddress; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/CreateKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/CreateKeyStoreCommand.java index dbc5d897417ee..05c1cfdc2fb9a 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/CreateKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/CreateKeyStoreCommand.java @@ -42,6 +42,7 @@ import org.opensearch.cli.KeyStoreAwareCommand; import org.opensearch.cli.Terminal; import org.opensearch.cli.UserException; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; /** diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 936fd032f36de..d06ffb650dc82 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -35,7 +35,7 @@ import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.test.OpenSearchTestCase; diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java index 7a0e286d2d961..15771945fa199 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java @@ -40,6 +40,7 @@ import org.apache.lucene.store.NIOFSDirectory; import org.opensearch.common.Randomness; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/ImportKeystoreTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/ImportKeystoreTask.java index c5d578dfad4df..5d35a7fcc0463 100644 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/ImportKeystoreTask.java +++ b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/ImportKeystoreTask.java @@ -12,7 +12,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.KeystoreWrapperUtil; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import java.io.InputStream; diff --git a/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java b/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java index 385dc24a0c25d..a03b2f94b27fe 100644 --- a/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java +++ b/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java @@ -38,14 +38,14 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Nullable; -import org.opensearch.common.compress.NotXContentException; +import org.opensearch.core.common.compress.NotXContentException; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.io.PrintWriter; diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java index 707b53dfa58f9..f75a225af1b4d 100644 --- a/libs/core/src/main/java/org/opensearch/OpenSearchException.java +++ b/libs/core/src/main/java/org/opensearch/OpenSearchException.java @@ -34,9 +34,9 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.logging.LoggerMessageFormat; @@ -45,9 +45,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; @@ -62,8 +62,8 @@ import static java.util.Collections.emptyMap; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static java.util.Collections.singletonMap; @@ -112,24 +112,24 @@ public class OpenSearchException extends RuntimeException implements Writeable, static { registerExceptionHandle( new OpenSearchExceptionHandle( - org.opensearch.index.snapshots.IndexShardSnapshotFailedException.class, - org.opensearch.index.snapshots.IndexShardSnapshotFailedException::new, + org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException.class, + org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException::new, 0, UNKNOWN_VERSION_ADDED ) ); registerExceptionHandle( new OpenSearchExceptionHandle( - org.opensearch.common.ParsingException.class, - org.opensearch.common.ParsingException::new, + org.opensearch.core.common.ParsingException.class, + org.opensearch.core.common.ParsingException::new, 40, UNKNOWN_VERSION_ADDED ) ); registerExceptionHandle( new OpenSearchExceptionHandle( - org.opensearch.common.io.stream.NotSerializableExceptionWrapper.class, - org.opensearch.common.io.stream.NotSerializableExceptionWrapper::new, + org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper.class, + org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper::new, 62, UNKNOWN_VERSION_ADDED ) diff --git a/libs/core/src/main/java/org/opensearch/common/io/package-info.java b/libs/core/src/main/java/org/opensearch/common/io/package-info.java deleted file mode 100644 index 0d57efe860d09..0000000000000 --- a/libs/core/src/main/java/org/opensearch/common/io/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** Classes for core io module */ -package org.opensearch.common.io; diff --git a/libs/core/src/main/java/org/opensearch/common/package-info.java b/libs/core/src/main/java/org/opensearch/common/package-info.java deleted file mode 100644 index 211cde81a3b88..0000000000000 --- a/libs/core/src/main/java/org/opensearch/common/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** common core classes that require third party dependencies */ -package org.opensearch.common; diff --git a/libs/core/src/main/java/org/opensearch/common/xcontent/package-info.java b/libs/core/src/main/java/org/opensearch/common/xcontent/package-info.java deleted file mode 100644 index c4522c2ae6627..0000000000000 --- a/libs/core/src/main/java/org/opensearch/common/xcontent/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** Base XContent Core Classes */ -package org.opensearch.common.xcontent; diff --git a/libs/core/src/main/java/org/opensearch/action/ShardOperationFailedException.java b/libs/core/src/main/java/org/opensearch/core/action/ShardOperationFailedException.java similarity index 95% rename from libs/core/src/main/java/org/opensearch/action/ShardOperationFailedException.java rename to libs/core/src/main/java/org/opensearch/core/action/ShardOperationFailedException.java index 35eaeb6e60eb0..9fdd4ae273a8b 100644 --- a/libs/core/src/main/java/org/opensearch/action/ShardOperationFailedException.java +++ b/libs/core/src/main/java/org/opensearch/core/action/ShardOperationFailedException.java @@ -30,12 +30,12 @@ * GitHub history for details. */ -package org.opensearch.action; +package org.opensearch.core.action; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.Objects; diff --git a/libs/core/src/main/java/org/opensearch/action/package-info.java b/libs/core/src/main/java/org/opensearch/core/action/package-info.java similarity index 86% rename from libs/core/src/main/java/org/opensearch/action/package-info.java rename to libs/core/src/main/java/org/opensearch/core/action/package-info.java index a7b89f63a6217..88cfb3326f8ed 100644 --- a/libs/core/src/main/java/org/opensearch/action/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/action/package-info.java @@ -7,4 +7,4 @@ */ /** Core action module */ -package org.opensearch.action; +package org.opensearch.core.action; diff --git a/libs/core/src/main/java/org/opensearch/action/support/DefaultShardOperationFailedException.java b/libs/core/src/main/java/org/opensearch/core/action/support/DefaultShardOperationFailedException.java similarity index 94% rename from libs/core/src/main/java/org/opensearch/action/support/DefaultShardOperationFailedException.java rename to libs/core/src/main/java/org/opensearch/core/action/support/DefaultShardOperationFailedException.java index 5a7e48a5efef7..ccb35ad5c6c6f 100644 --- a/libs/core/src/main/java/org/opensearch/action/support/DefaultShardOperationFailedException.java +++ b/libs/core/src/main/java/org/opensearch/core/action/support/DefaultShardOperationFailedException.java @@ -30,19 +30,19 @@ * GitHub history for details. */ -package org.opensearch.action.support; +package org.opensearch.core.action.support; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/action/support/package-info.java b/libs/core/src/main/java/org/opensearch/core/action/support/package-info.java similarity index 85% rename from libs/core/src/main/java/org/opensearch/action/support/package-info.java rename to libs/core/src/main/java/org/opensearch/core/action/support/package-info.java index 6665904e2b318..d0a1150d0750d 100644 --- a/libs/core/src/main/java/org/opensearch/action/support/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/action/support/package-info.java @@ -7,4 +7,4 @@ */ /** Support classes for core action module */ -package org.opensearch.action.support; +package org.opensearch.core.action.support; diff --git a/libs/core/src/main/java/org/opensearch/common/ParsingException.java b/libs/core/src/main/java/org/opensearch/core/common/ParsingException.java similarity index 95% rename from libs/core/src/main/java/org/opensearch/common/ParsingException.java rename to libs/core/src/main/java/org/opensearch/core/common/ParsingException.java index 5a9e99f5987d9..d2cd7a3170792 100644 --- a/libs/core/src/main/java/org/opensearch/common/ParsingException.java +++ b/libs/core/src/main/java/org/opensearch/core/common/ParsingException.java @@ -30,15 +30,15 @@ * GitHub history for details. */ -package org.opensearch.common; +package org.opensearch.core.common; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java rename to libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java index 043d45223498e..e054776d67fdc 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java @@ -29,12 +29,12 @@ * GitHub history for details. */ -package org.opensearch.common.bytes; +package org.opensearch.core.common.bytes; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.UnicodeUtil; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.EOFException; diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/BytesArray.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java similarity index 97% rename from libs/core/src/main/java/org/opensearch/common/bytes/BytesArray.java rename to libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java index aac34b991d186..ae04ddcc19eee 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/BytesArray.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java @@ -30,10 +30,10 @@ * GitHub history for details. */ -package org.opensearch.common.bytes; +package org.opensearch.core.common.bytes; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.io.OutputStream; diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java similarity index 97% rename from libs/core/src/main/java/org/opensearch/common/bytes/BytesReference.java rename to libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java index 5bafd24e1e928..fc8e62c914e27 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/BytesReference.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java @@ -30,14 +30,14 @@ * GitHub history for details. */ -package org.opensearch.common.bytes; +package org.opensearch.core.common.bytes; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.common.io.stream.BytesStream; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/CompositeBytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/common/bytes/CompositeBytesReference.java rename to libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java index 2d505d14124f2..53915a3da824c 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/CompositeBytesReference.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.bytes; +package org.opensearch.core.common.bytes; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/PagedBytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/PagedBytesReference.java similarity index 98% rename from libs/core/src/main/java/org/opensearch/common/bytes/PagedBytesReference.java rename to libs/core/src/main/java/org/opensearch/core/common/bytes/PagedBytesReference.java index 9d432bc6fdd8c..6b2ae821d2841 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/PagedBytesReference.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/PagedBytesReference.java @@ -30,11 +30,11 @@ * GitHub history for details. */ -package org.opensearch.common.bytes; +package org.opensearch.core.common.bytes; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/common/bytes/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/package-info.java similarity index 85% rename from libs/core/src/main/java/org/opensearch/common/bytes/package-info.java rename to libs/core/src/main/java/org/opensearch/core/common/bytes/package-info.java index 0644188656496..d197e07f3e5fe 100644 --- a/libs/core/src/main/java/org/opensearch/common/bytes/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core Bytes module */ -package org.opensearch.common.bytes; +package org.opensearch.core.common.bytes; diff --git a/libs/core/src/main/java/org/opensearch/common/compress/NotXContentException.java b/libs/core/src/main/java/org/opensearch/core/common/compress/NotXContentException.java similarity index 96% rename from libs/core/src/main/java/org/opensearch/common/compress/NotXContentException.java rename to libs/core/src/main/java/org/opensearch/core/common/compress/NotXContentException.java index 3859b267fbf74..d1a3e7709a7d0 100644 --- a/libs/core/src/main/java/org/opensearch/common/compress/NotXContentException.java +++ b/libs/core/src/main/java/org/opensearch/core/common/compress/NotXContentException.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.compress; +package org.opensearch.core.common.compress; import org.opensearch.core.xcontent.XContent; diff --git a/libs/core/src/main/java/org/opensearch/common/compress/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/compress/package-info.java similarity index 84% rename from libs/core/src/main/java/org/opensearch/common/compress/package-info.java rename to libs/core/src/main/java/org/opensearch/core/common/compress/package-info.java index b3b2f2a2ec7ed..99459f99c42d8 100644 --- a/libs/core/src/main/java/org/opensearch/common/compress/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/common/compress/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core compress module */ -package org.opensearch.common.compress; +package org.opensearch.core.common.compress; diff --git a/server/src/main/java/org/opensearch/common/io/stream/ByteBufferStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/ByteBufferStreamInput.java similarity index 98% rename from server/src/main/java/org/opensearch/common/io/stream/ByteBufferStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/ByteBufferStreamInput.java index 707b32a0c50f3..fcbab461b14c3 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/ByteBufferStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/ByteBufferStreamInput.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.EOFException; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/BytesStream.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStream.java similarity index 92% rename from libs/core/src/main/java/org/opensearch/common/io/stream/BytesStream.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStream.java index d9e1b19cccae7..70bf9c7710e2a 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/BytesStream.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStream.java @@ -30,9 +30,9 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; /** * Base Bytes Stream. diff --git a/server/src/main/java/org/opensearch/common/io/stream/BytesStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java similarity index 98% rename from server/src/main/java/org/opensearch/common/io/stream/BytesStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java index 8bf1fe846cd8b..a50d1c165ed72 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/BytesStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.util.BytesRef; diff --git a/server/src/main/java/org/opensearch/common/io/stream/DataOutputStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/DataOutputStreamOutput.java similarity index 97% rename from server/src/main/java/org/opensearch/common/io/stream/DataOutputStreamOutput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/DataOutputStreamOutput.java index e9b0b8d1f8876..b0bd6b101dc70 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/DataOutputStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/DataOutputStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.Closeable; import java.io.DataOutput; diff --git a/server/src/main/java/org/opensearch/common/io/stream/FilterStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java similarity index 98% rename from server/src/main/java/org/opensearch/common/io/stream/FilterStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java index 5f6bbd7c16cf3..a6e49567ac7d5 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/FilterStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.opensearch.Version; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/InputStreamStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/InputStreamStreamInput.java similarity index 98% rename from libs/core/src/main/java/org/opensearch/common/io/stream/InputStreamStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/InputStreamStreamInput.java index 06e10c67cf972..e86ad44fb1dc4 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/InputStreamStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/InputStreamStreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.EOFException; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/NamedWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteable.java similarity index 96% rename from libs/core/src/main/java/org/opensearch/common/io/stream/NamedWriteable.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteable.java index 380d764b60f9f..ae6aa5a9e9a17 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/NamedWriteable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteable.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; /** * A {@link Writeable} object identified by its name. diff --git a/server/src/main/java/org/opensearch/common/io/stream/NamedWriteableAwareStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableAwareStreamInput.java similarity index 98% rename from server/src/main/java/org/opensearch/common/io/stream/NamedWriteableAwareStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableAwareStreamInput.java index cf537a59edbdf..298090a18dec6 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/NamedWriteableAwareStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableAwareStreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/NamedWriteableRegistry.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java similarity index 97% rename from libs/core/src/main/java/org/opensearch/common/io/stream/NamedWriteableRegistry.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java index 985b2ad113f22..ec707f147cade 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/NamedWriteableRegistry.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.util.ArrayList; import java.util.Collections; @@ -40,7 +40,7 @@ import java.util.Objects; /** - * A registry for {@link org.opensearch.common.io.stream.Writeable.Reader} readers of {@link NamedWriteable}. + * A registry for {@link Writeable.Reader} readers of {@link NamedWriteable}. * * The registration is keyed by the combination of the category class of {@link NamedWriteable}, and a name unique * to that category. diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/NotSerializableExceptionWrapper.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NotSerializableExceptionWrapper.java similarity index 97% rename from libs/core/src/main/java/org/opensearch/common/io/stream/NotSerializableExceptionWrapper.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/NotSerializableExceptionWrapper.java index 967957afc8c20..91d552be0950c 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/NotSerializableExceptionWrapper.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NotSerializableExceptionWrapper.java @@ -30,11 +30,11 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/io/stream/OutputStreamStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/OutputStreamStreamOutput.java similarity index 97% rename from server/src/main/java/org/opensearch/common/io/stream/OutputStreamStreamOutput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/OutputStreamStreamOutput.java index 44cbe13be9aa7..cf45e0e0b9088 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/OutputStreamStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/OutputStreamStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.IOException; import java.io.OutputStream; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/common/io/stream/StreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java index c6915ae1f45b0..24795204c6dc8 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/StreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -46,10 +46,10 @@ import org.opensearch.Version; import org.opensearch.common.CharArrays; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.settings.SecureString; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.settings.SecureString; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; @@ -934,7 +934,7 @@ public byte[] readByteArray() throws IOException { } /** - * Reads an array from the stream using the specified {@link org.opensearch.common.io.stream.Writeable.Reader} to read array elements + * Reads an array from the stream using the specified {@link Writeable.Reader} to read array elements * from the stream. This method can be seen as the reader version of {@link StreamOutput#writeArray(Writeable.Writer, Object[])}. It is * assumed that the stream first contains a variable-length integer representing the size of the array, and then contains that many * elements that can be read from the stream. diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/common/io/stream/StreamOutput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index 888c7530c4146..566abf9f08f53 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -45,12 +45,12 @@ import org.opensearch.Version; import org.opensearch.common.CharArrays; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable.WriteableRegistry; -import org.opensearch.common.io.stream.Writeable.Writer; -import org.opensearch.common.settings.SecureString; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable.WriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable.Writer; +import org.opensearch.core.common.settings.SecureString; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; diff --git a/server/src/main/java/org/opensearch/common/io/stream/VersionedNamedWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VersionedNamedWriteable.java similarity index 97% rename from server/src/main/java/org/opensearch/common/io/stream/VersionedNamedWriteable.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/VersionedNamedWriteable.java index 408a2dc26386e..e7124e44692be 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/VersionedNamedWriteable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VersionedNamedWriteable.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.opensearch.Version; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/Writeable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/common/io/stream/Writeable.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java index 735413d3642ec..af9df51655414 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/Writeable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.IOException; import java.util.Map; diff --git a/libs/core/src/main/java/org/opensearch/common/io/stream/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/package-info.java similarity index 84% rename from libs/core/src/main/java/org/opensearch/common/io/stream/package-info.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/package-info.java index c332f90371f60..371f7ff7dd7b4 100644 --- a/libs/core/src/main/java/org/opensearch/common/io/stream/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/package-info.java @@ -7,4 +7,4 @@ */ /** Core transport stream classes */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; diff --git a/libs/core/src/main/java/org/opensearch/common/settings/SecureString.java b/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/common/settings/SecureString.java rename to libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java index 468d760766b96..f5529bcebc82f 100644 --- a/libs/core/src/main/java/org/opensearch/common/settings/SecureString.java +++ b/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.settings; +package org.opensearch.core.common.settings; import java.io.Closeable; import java.util.Arrays; diff --git a/libs/core/src/main/java/org/opensearch/common/settings/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/settings/package-info.java similarity index 84% rename from libs/core/src/main/java/org/opensearch/common/settings/package-info.java rename to libs/core/src/main/java/org/opensearch/core/common/settings/package-info.java index 10c1d5bab5fc1..a2588a4633417 100644 --- a/libs/core/src/main/java/org/opensearch/common/settings/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/common/settings/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core Settings module */ -package org.opensearch.common.settings; +package org.opensearch.core.common.settings; diff --git a/libs/core/src/main/java/org/opensearch/common/text/Text.java b/libs/core/src/main/java/org/opensearch/core/common/text/Text.java similarity index 96% rename from libs/core/src/main/java/org/opensearch/common/text/Text.java rename to libs/core/src/main/java/org/opensearch/core/common/text/Text.java index 36c64e6cec750..ca5402edae59e 100644 --- a/libs/core/src/main/java/org/opensearch/common/text/Text.java +++ b/libs/core/src/main/java/org/opensearch/core/common/text/Text.java @@ -29,11 +29,11 @@ * GitHub history for details. */ -package org.opensearch.common.text; +package org.opensearch.core.common.text; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/libs/core/src/main/java/org/opensearch/common/text/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/text/package-info.java similarity index 85% rename from libs/core/src/main/java/org/opensearch/common/text/package-info.java rename to libs/core/src/main/java/org/opensearch/core/common/text/package-info.java index d9ce2ebbc9340..02abc5835c84f 100644 --- a/libs/core/src/main/java/org/opensearch/common/text/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/common/text/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core text module */ -package org.opensearch.common.text; +package org.opensearch.core.common.text; diff --git a/libs/core/src/main/java/org/opensearch/common/util/BigArray.java b/libs/core/src/main/java/org/opensearch/core/common/util/BigArray.java similarity index 97% rename from libs/core/src/main/java/org/opensearch/common/util/BigArray.java rename to libs/core/src/main/java/org/opensearch/core/common/util/BigArray.java index f6cac6374bbeb..eafefa38bc061 100644 --- a/libs/core/src/main/java/org/opensearch/common/util/BigArray.java +++ b/libs/core/src/main/java/org/opensearch/core/common/util/BigArray.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.util; +package org.opensearch.core.common.util; import org.apache.lucene.util.Accountable; import org.opensearch.common.lease.Releasable; diff --git a/libs/core/src/main/java/org/opensearch/common/util/ByteArray.java b/libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java similarity index 98% rename from libs/core/src/main/java/org/opensearch/common/util/ByteArray.java rename to libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java index 44deb7f4fdb5a..e50f24417f8bc 100644 --- a/libs/core/src/main/java/org/opensearch/common/util/ByteArray.java +++ b/libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.util; +package org.opensearch.core.common.util; import org.apache.lucene.util.BytesRef; diff --git a/libs/core/src/main/java/org/opensearch/common/util/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/util/package-info.java similarity index 85% rename from libs/core/src/main/java/org/opensearch/common/util/package-info.java rename to libs/core/src/main/java/org/opensearch/core/common/util/package-info.java index f106d27c21881..57b1bfece1f90 100644 --- a/libs/core/src/main/java/org/opensearch/common/util/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/common/util/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core util module */ -package org.opensearch.common.util; +package org.opensearch.core.common.util; diff --git a/libs/core/src/main/java/org/opensearch/index/Index.java b/libs/core/src/main/java/org/opensearch/core/index/Index.java similarity index 95% rename from libs/core/src/main/java/org/opensearch/index/Index.java rename to libs/core/src/main/java/org/opensearch/core/index/Index.java index 769ca12958e3b..fc5c5152a5500 100644 --- a/libs/core/src/main/java/org/opensearch/index/Index.java +++ b/libs/core/src/main/java/org/opensearch/core/index/Index.java @@ -30,12 +30,12 @@ * GitHub history for details. */ -package org.opensearch.index; +package org.opensearch.core.index; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/libs/core/src/main/java/org/opensearch/index/package-info.java b/libs/core/src/main/java/org/opensearch/core/index/package-info.java similarity index 87% rename from libs/core/src/main/java/org/opensearch/index/package-info.java rename to libs/core/src/main/java/org/opensearch/core/index/package-info.java index bcb93e8c78856..31ecb88948c9d 100644 --- a/libs/core/src/main/java/org/opensearch/index/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/index/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core index module */ -package org.opensearch.index; +package org.opensearch.core.index; diff --git a/libs/core/src/main/java/org/opensearch/index/shard/ShardId.java b/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java similarity index 94% rename from libs/core/src/main/java/org/opensearch/index/shard/ShardId.java rename to libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java index ac24130ed90c9..b01121c3f30d4 100644 --- a/libs/core/src/main/java/org/opensearch/index/shard/ShardId.java +++ b/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java @@ -30,15 +30,15 @@ * GitHub history for details. */ -package org.opensearch.index.shard; +package org.opensearch.core.index.shard; import org.opensearch.core.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/index/shard/package-info.java b/libs/core/src/main/java/org/opensearch/core/index/shard/package-info.java similarity index 85% rename from libs/core/src/main/java/org/opensearch/index/shard/package-info.java rename to libs/core/src/main/java/org/opensearch/core/index/shard/package-info.java index 3ae1d95494bd6..267d558a5092c 100644 --- a/libs/core/src/main/java/org/opensearch/index/shard/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/index/shard/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core index/shard module */ -package org.opensearch.index.shard; +package org.opensearch.core.index.shard; diff --git a/libs/core/src/main/java/org/opensearch/index/snapshots/IndexShardSnapshotException.java b/libs/core/src/main/java/org/opensearch/core/index/snapshots/IndexShardSnapshotException.java similarity index 91% rename from libs/core/src/main/java/org/opensearch/index/snapshots/IndexShardSnapshotException.java rename to libs/core/src/main/java/org/opensearch/core/index/snapshots/IndexShardSnapshotException.java index 0d8001ec3d730..24e91ca949551 100644 --- a/libs/core/src/main/java/org/opensearch/index/snapshots/IndexShardSnapshotException.java +++ b/libs/core/src/main/java/org/opensearch/core/index/snapshots/IndexShardSnapshotException.java @@ -30,11 +30,11 @@ * GitHub history for details. */ -package org.opensearch.index.snapshots; +package org.opensearch.core.index.snapshots; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/index/snapshots/IndexShardSnapshotFailedException.java b/libs/core/src/main/java/org/opensearch/core/index/snapshots/IndexShardSnapshotFailedException.java similarity index 91% rename from libs/core/src/main/java/org/opensearch/index/snapshots/IndexShardSnapshotFailedException.java rename to libs/core/src/main/java/org/opensearch/core/index/snapshots/IndexShardSnapshotFailedException.java index db03dfc7f421c..32d1a49c46f9e 100644 --- a/libs/core/src/main/java/org/opensearch/index/snapshots/IndexShardSnapshotFailedException.java +++ b/libs/core/src/main/java/org/opensearch/core/index/snapshots/IndexShardSnapshotFailedException.java @@ -30,10 +30,10 @@ * GitHub history for details. */ -package org.opensearch.index.snapshots; +package org.opensearch.core.index.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/libs/core/src/main/java/org/opensearch/index/snapshots/package-info.java b/libs/core/src/main/java/org/opensearch/core/index/snapshots/package-info.java similarity index 84% rename from libs/core/src/main/java/org/opensearch/index/snapshots/package-info.java rename to libs/core/src/main/java/org/opensearch/core/index/snapshots/package-info.java index 7102459987dbf..b3bb62e40b7d0 100644 --- a/libs/core/src/main/java/org/opensearch/index/snapshots/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/index/snapshots/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core snapshots module */ -package org.opensearch.index.snapshots; +package org.opensearch.core.index.snapshots; diff --git a/libs/core/src/main/java/org/opensearch/rest/RestStatus.java b/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/rest/RestStatus.java rename to libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java index 8c718a5c8091c..ae4f4c65b28d2 100644 --- a/libs/core/src/main/java/org/opensearch/rest/RestStatus.java +++ b/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java @@ -30,11 +30,11 @@ * GitHub history for details. */ -package org.opensearch.rest; +package org.opensearch.core.rest; -import org.opensearch.action.ShardOperationFailedException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.action.ShardOperationFailedException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.HashMap; diff --git a/libs/core/src/main/java/org/opensearch/rest/package-info.java b/libs/core/src/main/java/org/opensearch/core/rest/package-info.java similarity index 87% rename from libs/core/src/main/java/org/opensearch/rest/package-info.java rename to libs/core/src/main/java/org/opensearch/core/rest/package-info.java index a629540e3858c..8c99d38e5f003 100644 --- a/libs/core/src/main/java/org/opensearch/rest/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/rest/package-info.java @@ -7,4 +7,4 @@ */ /** Classes for core REST module */ -package org.opensearch.rest; +package org.opensearch.core.rest; diff --git a/libs/core/src/main/java/org/opensearch/common/xcontent/XContentParserUtils.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java similarity index 96% rename from libs/core/src/main/java/org/opensearch/common/xcontent/XContentParserUtils.java rename to libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java index 5a91d9c2c1394..fff3d5f83f82e 100644 --- a/libs/core/src/main/java/org/opensearch/common/xcontent/XContentParserUtils.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java @@ -30,15 +30,12 @@ * GitHub history for details. */ -package org.opensearch.common.xcontent; +package org.opensearch.core.xcontent; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.Strings; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; -import org.opensearch.core.xcontent.XContentLocation; import java.io.IOException; import java.util.Locale; diff --git a/server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java b/libs/core/src/test/java/org/opensearch/core/action/support/DefaultShardOperationFailedExceptionTests.java similarity index 97% rename from server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java rename to libs/core/src/test/java/org/opensearch/core/action/support/DefaultShardOperationFailedExceptionTests.java index ee96ce5ad505c..d037b062e95d0 100644 --- a/server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java +++ b/libs/core/src/test/java/org/opensearch/core/action/support/DefaultShardOperationFailedExceptionTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.action.support; +package org.opensearch.core.action.support; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -40,17 +40,17 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import java.io.EOFException; diff --git a/server/src/test/java/org/opensearch/common/io/StreamsTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/StreamsTests.java similarity index 95% rename from server/src/test/java/org/opensearch/common/io/StreamsTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/StreamsTests.java index c52ce9703cc21..f2f46daa0fab3 100644 --- a/server/src/test/java/org/opensearch/common/io/StreamsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/StreamsTests.java @@ -30,11 +30,12 @@ * GitHub history for details. */ -package org.opensearch.common.io; +package org.opensearch.core.common.io; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.Streams; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayOutputStream; diff --git a/server/src/test/java/org/opensearch/common/io/stream/BaseStreamTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/BaseStreamTests.java similarity index 98% rename from server/src/test/java/org/opensearch/common/io/stream/BaseStreamTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/BaseStreamTests.java index bd970be5e977d..626900de7f691 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/BaseStreamTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/BaseStreamTests.java @@ -30,16 +30,17 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedBiConsumer; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.script.JodaCompatibleZonedDateTime; import org.opensearch.test.OpenSearchTestCase; @@ -188,6 +189,7 @@ public void testLinkedHashMap() throws IOException { } BytesStreamOutput out = new BytesStreamOutput(); out.writeGenericValue(write); + @SuppressWarnings("unchecked") LinkedHashMap read = (LinkedHashMap) getStreamInput(out.bytes()).readGenericValue(); assertEquals(size, read.size()); int index = 0; diff --git a/server/src/test/java/org/opensearch/common/io/stream/ByteBufferStreamInputTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/ByteBufferStreamInputTests.java similarity index 87% rename from server/src/test/java/org/opensearch/common/io/stream/ByteBufferStreamInputTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/ByteBufferStreamInputTests.java index 1061b335d715a..d50160b2b3e2d 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/ByteBufferStreamInputTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/ByteBufferStreamInputTests.java @@ -6,10 +6,10 @@ * compatible open source license. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/server/src/test/java/org/opensearch/common/io/stream/BytesReferenceStreamInputTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/BytesReferenceStreamInputTests.java similarity index 83% rename from server/src/test/java/org/opensearch/common/io/stream/BytesReferenceStreamInputTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/BytesReferenceStreamInputTests.java index ed77c3130a32d..7eca07d9b63d5 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/BytesReferenceStreamInputTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/BytesReferenceStreamInputTests.java @@ -6,9 +6,9 @@ * compatible open source license. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/io/stream/BytesStreamInputTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/BytesStreamInputTests.java similarity index 86% rename from server/src/test/java/org/opensearch/common/io/stream/BytesStreamInputTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/BytesStreamInputTests.java index c7a47e7580b02..d61eea75538b3 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/BytesStreamInputTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/BytesStreamInputTests.java @@ -6,10 +6,10 @@ * compatible open source license. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/io/stream/FilterStreamInputTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java similarity index 86% rename from server/src/test/java/org/opensearch/common/io/stream/FilterStreamInputTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java index 3cf9dc656a8f1..a044586e095e3 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/FilterStreamInputTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java @@ -6,10 +6,10 @@ * compatible open source license. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/io/stream/InputStreamStreamInputTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/InputStreamStreamInputTests.java similarity index 86% rename from server/src/test/java/org/opensearch/common/io/stream/InputStreamStreamInputTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/InputStreamStreamInputTests.java index 6a31c21445a04..4bcd697b1c283 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/InputStreamStreamInputTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/InputStreamStreamInputTests.java @@ -6,10 +6,10 @@ * compatible open source license. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/io/stream/NamedWriteableRegistryTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/NamedWriteableRegistryTests.java similarity index 98% rename from server/src/test/java/org/opensearch/common/io/stream/NamedWriteableRegistryTests.java rename to libs/core/src/test/java/org/opensearch/core/common/io/stream/NamedWriteableRegistryTests.java index e1c0177435005..77e5af9036cea 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/NamedWriteableRegistryTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/NamedWriteableRegistryTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import java.io.IOException; import java.util.Arrays; diff --git a/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java index d6191e3714149..421263b883f2a 100644 --- a/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java @@ -12,7 +12,7 @@ import org.apache.lucene.util.BytesRefArray; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.Counter; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.test.OpenSearchTestCase; import java.nio.ByteBuffer; diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/SslConfigurationLoaderTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/SslConfigurationLoaderTests.java index 19f4a85115236..13b708df497e0 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/SslConfigurationLoaderTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/SslConfigurationLoaderTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.ssl; import org.opensearch.common.settings.MockSecureSettings; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MapXContentParserTests.java index f15823481180b..ed6d38289ef6a 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MapXContentParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MapXContentParserTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.xcontent; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MapXContentParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/ObjectParserTests.java index 9e286e8deeb80..cd59bf59fe15d 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/ObjectParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/ObjectParserTests.java @@ -34,6 +34,7 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.NamedObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java index 2920a1ae6cf04..3552514af0aa8 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.cbor.CborXContent; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStats.java index c47a5a9ffc49e..3099ba3c6aa19 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStats.java +++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStats.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.matrix.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java index b3afe44cf64ba..f757968bd86f7 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java +++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.matrix.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsResults.java index 87eb58ab3427a..6e04efbf6689c 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsResults.java +++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsResults.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.matrix.stats; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Collections; diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java index cad0011602f67..de67cc2930652 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java +++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.matrix.stats; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.ArrayList; diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java index 967844b1c2208..102188deb91b5 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java +++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.support; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.AbstractAggregationBuilder; diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceParser.java index 051b731aa0aef..855e1ca8e92ec 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceParser.java +++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/support/ArrayValuesSourceParser.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.support; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; import org.opensearch.search.aggregations.AggregationBuilder.CommonFields; diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java index 8020b3cf19fee..3fa6f0fda406d 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java @@ -128,7 +128,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Settings; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CharGroupTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CharGroupTokenizerFactoryTests.java index 54ab1bc154641..95886cda8b025 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CharGroupTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CharGroupTokenizerFactoryTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.util.CharTokenizer; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.test.OpenSearchTokenStreamTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/EdgeNGramTokenizerTests.java index ae3ebde99bd98..6ac5f42d2a66f 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/EdgeNGramTokenizerTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/EdgeNGramTokenizerTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java index 7d9555923e4ef..d26949c30f203 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.test.OpenSearchTokenStreamTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java index 49ccc742a355c..6e4dc558fed99 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.test.OpenSearchTokenStreamTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java index 73c104a5b7200..12e428be6c821 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.analysis.Tokenizer; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTokenStreamTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java index c0034c0e8ef20..70f6269ad0d04 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java @@ -38,7 +38,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java index ed3196319faca..4be965b862ddf 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java @@ -35,7 +35,7 @@ import org.hamcrest.MatcherAssert; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.util.BigArray; +import org.opensearch.core.common.util.BigArray; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.terms.Terms; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index bb64da8d85512..50c93edacd6b7 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -35,8 +35,8 @@ import org.apache.lucene.index.IndexReader; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGrid.java index 9b5ed7777204e..d41518cedbf86 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGrid.java @@ -31,8 +31,8 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.LongObjectPagedHashMap; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGridBucket.java index d58c9752a85e9..e559c0995f86f 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BaseGeoGridBucket.java @@ -31,8 +31,8 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 7c6273a521097..c716f2d3b2306 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -36,8 +36,8 @@ import org.opensearch.core.ParseField; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGrid.java index 9b6713ac033ae..1954ba421e159 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGrid.java @@ -31,7 +31,7 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import java.io.IOException; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index b24f735629476..ce51946685f26 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoUtils; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.AggregationBuilder; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGrid.java index bf45080759a07..3be005334ced9 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGrid.java @@ -31,7 +31,7 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import java.io.IOException; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index e86d805ff6481..81bd85373e40b 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.AggregationBuilder; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java index 6e7ed8a679681..2286c6113c36d 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java @@ -32,7 +32,7 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.geometry.utils.Geohash; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java index b0f8dcc806c5b..0ef5937ff160d 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java @@ -33,7 +33,7 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.GeoTileUtils; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java index 1afd19f6db068..f959c6bd6842c 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java index 59f2b95dc57ed..c3cb47d244c6b 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java @@ -34,8 +34,8 @@ import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java index 1f0680b27796d..49c7c7ae152c0 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java @@ -33,8 +33,8 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ingest.IngestStats; diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessorGetAction.java index 49e66bb526401..300106e435114 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessorGetAction.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/GrokProcessorGetAction.java @@ -40,8 +40,8 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.grok.Grok; diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/JsonProcessor.java index a6b7e30f71458..d3a0f2f7a3783 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/JsonProcessor.java @@ -32,8 +32,8 @@ package org.opensearch.ingest.common; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ScriptProcessor.java index a7825edcf6b3a..5f61091495cd5 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ScriptProcessor.java @@ -33,7 +33,7 @@ package org.opensearch.ingest.common; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorGetActionTests.java index db8c04cca7c80..a76c194037224 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorGetActionTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/GrokProcessorGetActionTests.java @@ -34,9 +34,9 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/JsonProcessorTests.java index f2cc33e10abfe..531e22a386236 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/JsonProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/JsonProcessorTests.java @@ -32,7 +32,7 @@ package org.opensearch.ingest.common; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/opensearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/opensearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java index 6ec147c7390a9..b947f929fce76 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/opensearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/opensearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java @@ -36,7 +36,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.ingest.PutPipelineRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; @@ -44,7 +44,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.ingest.IngestService; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.NodeRoles; import org.opensearch.test.StreamsUtils; diff --git a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java index cf14b47fa9f3e..7527092f4344c 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java @@ -32,7 +32,7 @@ package org.opensearch.script.expression; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.plugins.Plugin; diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/SearchTemplateIT.java index 87ef8b810f5e0..d0a0b7bcd3e3e 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/SearchTemplateIT.java @@ -35,7 +35,7 @@ import org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.opensearch.action.bulk.BulkRequestBuilder; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.plugins.Plugin; diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateRequest.java index 35363275da304..dc3b52cd0b946 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateRequest.java @@ -38,9 +38,9 @@ import org.opensearch.action.search.MultiSearchRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java index 449e89cdc5f3c..f31e5be078a28 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java @@ -37,9 +37,9 @@ import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateRequest.java index 202ba065906d7..c963ea7ba7da9 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateRequest.java @@ -37,10 +37,10 @@ import org.opensearch.action.CompositeIndicesRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java index 87f3b9f902b8f..6e56ecf3950bb 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java @@ -35,16 +35,16 @@ import org.opensearch.action.ActionResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.io.InputStream; diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java index 199c01087d0fe..bbda8d15d9d41 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java @@ -38,7 +38,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateRequestTests.java index 1a663dcb18235..311f14cb8d80d 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.script.mustache; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.script.ScriptType; diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheTests.java index bc44821416fbe..a5c9a03590321 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheTests.java @@ -31,7 +31,7 @@ package org.opensearch.script.mustache; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.script.ScriptEngine; diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestTests.java index 6c8e91d8c4d17..72443d1323b44 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestTests.java @@ -32,7 +32,7 @@ package org.opensearch.script.mustache; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.script.ScriptType; import org.opensearch.search.RandomSearchRequestGenerator; import org.opensearch.search.builder.SearchSourceBuilder; diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java index 1e997eee9ff22..664e87da0a7d8 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java @@ -32,7 +32,7 @@ package org.opensearch.script.mustache; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java index 642625f932155..804a72561c10e 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.TotalHits; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java b/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java index dea84434e95e3..c7b9c75570899 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextAction.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextAction.java index e7d6a045ccc42..db1ef8f7a30fb 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextAction.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextAction.java @@ -42,9 +42,9 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.ParseField; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.painless.PainlessScriptEngine; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassBindingInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassBindingInfo.java index efd0c7a190194..0fa5cc5658b6a 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassBindingInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassBindingInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassInfo.java index eec518aca2dd3..25f63e1c8bdfa 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextClassInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextConstructorInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextConstructorInfo.java index 0a9b0cb815d77..cc1fc7448d134 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextConstructorInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextConstructorInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextFieldInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextFieldInfo.java index 05cbce272fd57..7187f3403795e 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextFieldInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextFieldInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInfo.java index 88d34994a4535..e4642b98154a5 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInstanceBindingInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInstanceBindingInfo.java index 071985d49eed8..848868f675422 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInstanceBindingInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextInstanceBindingInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextMethodInfo.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextMethodInfo.java index 5552d35b161c0..02061b4543d9f 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextMethodInfo.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessContextMethodInfo.java @@ -33,9 +33,9 @@ package org.opensearch.painless.action; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java index 18f854b3d1ea2..faae24083eeca 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java @@ -58,11 +58,11 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedBiFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -72,14 +72,14 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/DebugTests.java index 90bb6bac30f0d..c95a6f2f2604c 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/DebugTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/DebugTests.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.painless.lookup.PainlessLookup; import org.opensearch.painless.lookup.PainlessLookupBuilder; import org.opensearch.painless.spi.Allowlist; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/action/ContextInfoTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/action/ContextInfoTests.java index ac76a3e19daf3..b576a65db61df 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/action/ContextInfoTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/action/ContextInfoTests.java @@ -32,7 +32,7 @@ package org.opensearch.painless.action; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java index 6e318eda91985..e80cc2d23e290 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java @@ -31,7 +31,7 @@ package org.opensearch.painless.action; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexService; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteRequestTests.java index baa43625054aa..8204f9db0b8d6 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteRequestTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteRequestTests.java @@ -31,10 +31,10 @@ package org.opensearch.painless.action; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteResponseTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteResponseTests.java index fe9e454fdcbb9..9c9ba090685ec 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteResponseTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteResponseTests.java @@ -31,7 +31,7 @@ package org.opensearch.painless.action; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java b/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java index d1b6df3f91c85..3b0384ca7d62f 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java @@ -36,8 +36,8 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureMetaFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureMetaFieldMapperTests.java index 63487fd7baa89..63b1b4760b6fe 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureMetaFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureMetaFieldMapperTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java index 2662605cbfe5b..390f10c0684bd 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/ChildQuerySearchIT.java index cacd7c3a23824..6fdcd7f5488f4 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/ChildQuerySearchIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/ChildQuerySearchIT.java @@ -50,7 +50,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.bucket.filter.Filter; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ChildrenAggregationBuilder.java index e0a3fcf8203eb..b58e278133d95 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ChildrenAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ChildrenAggregationBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.join.aggregations; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalChildren.java b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalChildren.java index d8221d2d2afb1..dfe34d16d016b 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalChildren.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalChildren.java @@ -32,7 +32,7 @@ package org.opensearch.join.aggregations; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalParent.java b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalParent.java index d512d62526306..cd4522b3b063b 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalParent.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/InternalParent.java @@ -32,7 +32,7 @@ package org.opensearch.join.aggregations; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ParentAggregationBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ParentAggregationBuilder.java index dc587a8d59179..7d0aaee71583e 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ParentAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/aggregations/ParentAggregationBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.join.aggregations; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java index c29341e0523ab..4f26a15031dd4 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java @@ -43,9 +43,9 @@ import org.apache.lucene.search.similarities.Similarity; import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java index b0e608ebca105..97809af772979 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.join.ScoreMode; import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java index 23dbbe42a28be..bfc01bf151a9c 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java @@ -38,9 +38,9 @@ import org.apache.lucene.search.Query; import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; diff --git a/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ChildrenToParentAggregatorTests.java index d1b4f3c3ebc27..1d0f6ebdd8349 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -50,7 +50,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.IdFieldMapper; @@ -60,7 +60,7 @@ import org.opensearch.index.mapper.MappingLookup; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.Uid; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.join.ParentJoinModulePlugin; import org.opensearch.join.mapper.MetaJoinFieldMapper; import org.opensearch.join.mapper.ParentJoinFieldMapper; diff --git a/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ParentToChildrenAggregatorTests.java index 61a1b761f17ab..1e09d75dc7197 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ParentToChildrenAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/aggregations/ParentToChildrenAggregatorTests.java @@ -51,7 +51,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.MappingLookup; import org.opensearch.index.mapper.DocumentMapper; @@ -61,7 +61,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.Uid; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.join.ParentJoinModulePlugin; import org.opensearch.join.mapper.MetaJoinFieldMapper; import org.opensearch.join.mapper.ParentJoinFieldMapper; diff --git a/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java index 53e9495b707fe..cd8f18b679750 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java @@ -33,7 +33,7 @@ package org.opensearch.join.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java index a0b04f62a84b6..278a8c1cc0ebc 100644 --- a/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java @@ -36,8 +36,8 @@ import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.DistanceUnit; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java index 0aa8318e7c191..86d30f009e709 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java @@ -50,7 +50,7 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; import java.io.IOException; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java index 498b58c8b6bba..d5b61d5c5a517 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java @@ -60,12 +60,12 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.get.GetRequest; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java index b3c23dc455e28..a2fb9e56fd9f0 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java @@ -56,11 +56,11 @@ import org.apache.lucene.util.BytesRefBuilder; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.hash.MurmurHash3; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhase.java index 9b4e42d239750..91c0d2fad44b4 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -36,7 +36,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.search.SearchHit; import org.opensearch.search.fetch.FetchContext; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java index 1e884f591cbf8..3b953fcfe65e1 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java @@ -95,8 +95,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedFunction; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.common.geo.ShapeRelation; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryBuilderTests.java index 5014b838d94e3..34ed195cd0f23 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryBuilderTests.java @@ -39,8 +39,8 @@ import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.uid.Versions; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryTests.java index c5049e21acc0c..f75367fc054c0 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolateQueryTests.java @@ -56,7 +56,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolateWithNestedQueryBuilderTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolateWithNestedQueryBuilderTests.java index 0ab9eff731bff..748b79d70af07 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolateWithNestedQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolateWithNestedQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.mapper.MapperService; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java index eb76229801479..77a4718b1d755 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java @@ -57,15 +57,15 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.hash.MurmurHash3; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhaseTests.java index 83ca9037658b5..08c48e430119a 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorHighlightSubFetchPhaseTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.common.lucene.search.function.RandomScoreFunction; import org.opensearch.search.fetch.FetchContext; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java index 1e85e1ad18314..7f0437a8a29aa 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java index 9c7aa4b2f300a..b93b0427e1f67 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java @@ -45,7 +45,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.fielddata.plain.BytesBinaryIndexFieldData; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/DiscountedCumulativeGain.java index 5c7a4137fcb24..31c52cab404c6 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/DiscountedCumulativeGain.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvalQueryQuality.java index c1a629a81392d..b08f6bee571d5 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvalQueryQuality.java @@ -33,14 +33,14 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.rankeval.RatedDocument.DocumentKey; import java.io.IOException; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvaluationMetric.java index 5f704f1ba304d..5330121cd1295 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvaluationMetric.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/EvaluationMetric.java @@ -32,7 +32,7 @@ package org.opensearch.index.rankeval; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.index.rankeval.RatedDocument.DocumentKey; import org.opensearch.search.SearchHit; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/ExpectedReciprocalRank.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/ExpectedReciprocalRank.java index eeb273abe408c..6d7f90fb63659 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/ExpectedReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/ExpectedReciprocalRank.java @@ -34,8 +34,8 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MeanReciprocalRank.java index 8410d7fb20bdd..bc455f30f092b 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MeanReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MeanReciprocalRank.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MetricDetail.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MetricDetail.java index c60d2d3b1351a..5627ac37a47fc 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MetricDetail.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/MetricDetail.java @@ -32,7 +32,7 @@ package org.opensearch.index.rankeval; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/PrecisionAtK.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/PrecisionAtK.java index b9c8e02878df2..a10b957f57a6b 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/PrecisionAtK.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/PrecisionAtK.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalModulePlugin.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalModulePlugin.java index cd6606b96c4d2..3dbc07e5cf342 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalModulePlugin.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalModulePlugin.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalRequest.java index b3975174dd69e..ba640a96d2642 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalRequest.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalRequest.java @@ -38,8 +38,8 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchType; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalResponse.java index 180069ab91bd6..4dbc348fe458e 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalResponse.java @@ -37,13 +37,13 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalSpec.java index d200af0c19087..9585e79e69cf1 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalSpec.java @@ -33,16 +33,16 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.script.Script; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedDocument.java index 63e05983409e3..02ac9182c4f35 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedDocument.java @@ -33,9 +33,9 @@ package org.opensearch.index.rankeval; import org.opensearch.core.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedRequest.java index 094b26e3d14f5..c5f899cbefdf6 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedRequest.java @@ -34,9 +34,9 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedSearchHit.java index 24ea3bcad7ecf..f7b2b7a890334 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedSearchHit.java @@ -33,9 +33,9 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.ToXContent; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RecallAtK.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RecallAtK.java index ca56eeeee7d3c..fdbdead7575a7 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RecallAtK.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RecallAtK.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java index f22fa17b3904e..688a0787c95b8 100644 --- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java @@ -40,9 +40,9 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.client.Client; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java index 1de854b91a6ae..ea001de0ee7c6 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -34,8 +34,8 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -43,7 +43,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/EvalQueryQualityTests.java index 078134a737308..2b99e41b8267c 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/EvalQueryQualityTests.java @@ -33,13 +33,13 @@ package org.opensearch.index.rankeval; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; @@ -49,7 +49,7 @@ import java.util.function.Predicate; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java index c7573bc50e86b..014f52faa9d57 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -42,7 +42,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java index fc745ac193eea..3df79acfa6ce1 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -42,7 +42,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java index 020086f807900..3317a2d2f00f1 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -42,7 +42,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalRequestTests.java index 37bf4e31e25d0..a13b1d0511e68 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalRequestTests.java @@ -34,8 +34,8 @@ import org.opensearch.action.search.SearchType; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.common.util.ArrayUtils; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java index b52c1c1897086..7c0590566bba9 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java @@ -38,19 +38,19 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.coordination.NoClusterManagerBlockService; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchParseException; import org.opensearch.search.SearchShardTarget; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalSpecTests.java index e4a21b61052cd..c42c0722e0fae 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalSpecTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java index 909024e2abe86..988784b6e57a3 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.rankeval; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java index 9de79f246c5e5..ac2401f30e6f0 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.rankeval; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedSearchHitTests.java index e8453bbc4b9fb..791d3d03cd939 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedSearchHitTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.rankeval; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java index 825436f02ae2b..85e024f6bb1e9 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.rankeval; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -42,7 +42,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/reindex/src/internalClusterTest/java/org/opensearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/internalClusterTest/java/org/opensearch/client/documentation/ReindexDocumentationIT.java index 05cb87b3f9165..3a19be57cf731 100644 --- a/modules/reindex/src/internalClusterTest/java/org/opensearch/client/documentation/ReindexDocumentationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/opensearch/client/documentation/ReindexDocumentationIT.java @@ -53,7 +53,7 @@ import org.opensearch.index.reindex.UpdateByQueryAction; import org.opensearch.index.reindex.UpdateByQueryRequestBuilder; import org.opensearch.index.shard.IndexingOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.plugins.Plugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 207a203114316..6170c1adabbea 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -92,7 +92,7 @@ import static org.opensearch.action.bulk.BackoffPolicy.exponentialBackoff; import static org.opensearch.common.unit.TimeValue.timeValueNanos; import static org.opensearch.index.reindex.AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES; -import static org.opensearch.rest.RestStatus.CONFLICT; +import static org.opensearch.core.rest.RestStatus.CONFLICT; import static org.opensearch.search.sort.SortBuilders.fieldSort; /** diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBaseReindexRestHandler.java index d029ae0be9836..f034e92551ede 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBaseReindexRestHandler.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.tasks.LoggingTaskListener; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java index 7367182150aea..6f36fafd852ef 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionType; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java index 334390ca730cf..fcdee900b4ca7 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -39,7 +39,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.client.Client; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.slice.SliceBuilder; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkIndexByScrollResponseContentListener.java index f7f8721595e05..1ed61c200ce28 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkIndexByScrollResponseContentListener.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkIndexByScrollResponseContentListener.java @@ -39,7 +39,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.util.Map; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexModulePlugin.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexModulePlugin.java index 1a64eb98c7ac1..d89e3d677a2a5 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexModulePlugin.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexModulePlugin.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexSslConfig.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexSslConfig.java index 0e0e387b78e38..10b94efe4d868 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexSslConfig.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexSslConfig.java @@ -39,7 +39,7 @@ import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.reactor.ssl.TlsDetails; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.ssl.SslConfiguration; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java index a5107597e4113..ba974d248ac9b 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java @@ -57,7 +57,7 @@ import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/RestDeleteByQueryAction.java index 6f2e5d8e71edb..4f68884b194e9 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/RestDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/RestDeleteByQueryAction.java @@ -33,7 +33,7 @@ package org.opensearch.index.reindex; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.rest.RestRequest; import java.io.IOException; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/RestReindexAction.java index 662fa1f84c9a5..c03eba10b41ba 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/RestReindexAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/RestReindexAction.java @@ -34,7 +34,7 @@ import org.opensearch.action.DocWriteRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestRequestFilter; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/RestUpdateByQueryAction.java index 9be1687a09432..aeb339d49e1bb 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/RestUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/RestUpdateByQueryAction.java @@ -33,7 +33,7 @@ package org.opensearch.index.reindex; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.rest.RestRequest; import org.opensearch.script.Script; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/RethrottleRequest.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/RethrottleRequest.java index f174aa6d6fda3..2036479c73bc4 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/RethrottleRequest.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/RethrottleRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.tasks.BaseTasksRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java index 51e388227eb10..50dbc972061a5 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java @@ -39,7 +39,7 @@ import org.opensearch.client.ParentTaskAssigningClient; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.script.ScriptService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java index f07915b9d9e76..5297723ab3687 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.IndexFieldMapper; import org.opensearch.index.mapper.RoutingFieldMapper; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteRequestBuilders.java index 970407bb1748e..511c44ae3c2b6 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteRequestBuilders.java @@ -39,7 +39,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.client.Request; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java index 5a4674e146585..3d606b6b77c44 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.TotalHits; import org.opensearch.LegacyESVersion; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.collect.Tuple; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java index d3049fff07192..39d3adcd82ad5 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -50,7 +50,7 @@ import org.opensearch.client.ResponseListener; import org.opensearch.client.RestClient; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -61,7 +61,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.reindex.RejectAwareActionListener; import org.opensearch.index.reindex.ScrollableHitSource; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java index 61336156a83d0..deb78edf2f01e 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -71,18 +71,18 @@ import org.opensearch.client.ParentTaskAssigningClient; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.index.reindex.ScrollableHitSource.Hit; import org.opensearch.index.reindex.ScrollableHitSource.SearchFailure; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.internal.InternalSearchResponse; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/CancelTests.java index bd43f05225f65..26fcfd226371f 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/CancelTests.java @@ -39,15 +39,15 @@ import org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.opensearch.action.ingest.DeletePipelineRequest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexModule; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.Engine.Operation.Origin; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.shard.IndexingOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.ingest.IngestTestPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.tasks.TaskCancelledException; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java index 25917893645cf..46113971d537a 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java @@ -45,7 +45,7 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.client.ParentTaskAssigningClient; import org.opensearch.client.support.AbstractClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteAllowlistTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteAllowlistTests.java index 7ade56e378ac0..1f8fe7bf79d46 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteAllowlistTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteAllowlistTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.reindex; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.test.OpenSearchTestCase; import java.net.UnknownHostException; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java index bd2aae5b7a7ab..0355bfcab02de 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -34,8 +34,8 @@ import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilderTestCase; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java index ce15d59b7936c..97f43b9439408 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java @@ -46,8 +46,8 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; @@ -60,7 +60,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestHeaderDefinition; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.ScriptService; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexRestClientSslTests.java index eb19454d8d7e8..4bde5f353f09e 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexRestClientSslTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexRestClientSslTests.java @@ -40,7 +40,7 @@ import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.ssl.PemKeyConfig; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexSourceTargetValidationTests.java index e3bd1c1cba4a6..a554c25a47f65 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexSourceTargetValidationTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexSourceTargetValidationTests.java @@ -44,8 +44,8 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.indices.SystemIndices; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java index 4b9aa0ac2ce62..2946544a8f901 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java @@ -32,9 +32,9 @@ package org.opensearch.index.reindex; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java index 4dbd18a15b521..e239018e0ce31 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.bulk.Retry; import org.opensearch.client.Client; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RoundTripTests.java index edd301603250a..4508bf59d6879 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RoundTripTests.java @@ -33,11 +33,11 @@ package org.opensearch.index.reindex; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.TimeValue; import org.opensearch.script.Script; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteInfoTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteInfoTests.java index 91558963d4343..640a09c33b3a6 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteInfoTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteInfoTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.reindex.remote; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.index.reindex.RemoteInfo; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteRequestBuildersTests.java index e7af54a0563d3..54e455d9e676e 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteRequestBuildersTests.java @@ -37,8 +37,8 @@ import org.opensearch.Version; import org.opensearch.action.search.SearchRequest; import org.opensearch.client.Request; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.Streams; import org.opensearch.common.unit.TimeValue; import org.opensearch.search.builder.SearchSourceBuilder; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java index 2c5b6eaf52984..ebbd2da776ace 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java @@ -40,8 +40,8 @@ import org.opensearch.client.RestClient; import org.opensearch.client.http.HttpUriRequestProducer; import org.opensearch.client.nio.HeapBufferedAsyncResponseConsumer; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.common.io.Streams; import org.opensearch.common.unit.ByteSizeUnit; @@ -52,7 +52,7 @@ import org.opensearch.index.reindex.RejectAwareActionListener; import org.opensearch.index.reindex.ScrollableHitSource; import org.opensearch.index.reindex.ScrollableHitSource.Response; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/modules/repository-url/src/test/java/org/opensearch/repositories/url/URLFixture.java b/modules/repository-url/src/test/java/org/opensearch/repositories/url/URLFixture.java index 04c2d249be635..f9f2ecaca486d 100644 --- a/modules/repository-url/src/test/java/org/opensearch/repositories/url/URLFixture.java +++ b/modules/repository-url/src/test/java/org/opensearch/repositories/url/URLFixture.java @@ -33,7 +33,7 @@ import org.opensearch.test.fixture.AbstractHttpFixture; import org.opensearch.common.SuppressForbidden; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.nio.file.Files; diff --git a/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java index fe9799bb5e13e..705cbafd1bd3a 100644 --- a/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java +++ b/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -43,7 +43,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; import org.apache.hc.core5.http.ContentType; diff --git a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java index 2c7ffb4648a7d..1e24476eeff27 100644 --- a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java +++ b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java @@ -18,13 +18,13 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java index c776181a64b26..eab8bc95bd668 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java @@ -9,7 +9,7 @@ package org.opensearch.search.pipeline.common; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java index c410895304f76..f959bccc93c20 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessor.java @@ -10,7 +10,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.common.xcontent.XContentHelper; diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java index e100fe0e3edce..3849d2f905490 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java @@ -11,7 +11,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java index b7d1510d065ae..f50d29ff3be6c 100644 --- a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/RenameFieldResponseProcessorTests.java @@ -13,7 +13,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchResponseSections; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.document.DocumentField; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.TermQueryBuilder; diff --git a/modules/systemd/src/main/java/org/opensearch/systemd/SystemdModulePlugin.java b/modules/systemd/src/main/java/org/opensearch/systemd/SystemdModulePlugin.java index 2919e83a00598..eb04e69810b9c 100644 --- a/modules/systemd/src/main/java/org/opensearch/systemd/SystemdModulePlugin.java +++ b/modules/systemd/src/main/java/org/opensearch/systemd/SystemdModulePlugin.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; diff --git a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java index 3f504ff36b148..2584b768707cd 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java @@ -47,7 +47,7 @@ import java.nio.charset.Charset; import java.util.Map; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; import static org.opensearch.test.hamcrest.RegexMatcher.matches; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4HeadBodyIsEmptyIT.java index af984f5e416e8..4b962401387b7 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4HeadBodyIsEmptyIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4HeadBodyIsEmptyIT.java @@ -45,8 +45,8 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; import static org.hamcrest.Matchers.greaterThan; public class Netty4HeadBodyIsEmptyIT extends OpenSearchRestTestCase { diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java index 8ce3af0bb141c..c502cb9882b55 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java @@ -43,10 +43,10 @@ import io.netty.handler.codec.http.cookie.Cookie; import io.netty.handler.codec.http.cookie.ServerCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.http.HttpRequest; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.transport.netty4.Netty4Utils; import java.util.AbstractMap; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpResponse.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpResponse.java index 78df964ce9cac..c1fd97022d1af 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpResponse.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpResponse.java @@ -36,9 +36,9 @@ import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpVersion; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.http.HttpResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.transport.netty4.Netty4Utils; public class Netty4HttpResponse extends DefaultFullHttpResponse implements HttpResponse { diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java index 8bcd740889422..d7f2f6eb6acbb 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java index 5e01823bf808c..592ea45016bf3 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java @@ -40,7 +40,7 @@ import io.netty.channel.ChannelPromise; import org.opensearch.OpenSearchException; import org.opensearch.ExceptionsHelper; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.lease.Releasables; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java index eeee0ff33e792..46cbf60d45116 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java @@ -38,7 +38,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.concurrent.CompletableContext; import org.opensearch.transport.TcpChannel; import org.opensearch.transport.TransportException; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java index 5bf515548eb54..561cac2facbff 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java @@ -52,7 +52,7 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Utils.java index e13329b8c2593..10fd1cf4366c4 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Utils.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Utils.java @@ -39,8 +39,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.opensearch.common.Booleans; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java index c18fe6efc4736..ef014aa39367b 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java @@ -48,7 +48,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java index d37e28dabf19d..88dac2b2c6cdd 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java @@ -42,12 +42,12 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.QueryStringDecoder; import org.opensearch.common.Randomness; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.http.HttpPipelinedRequest; import org.opensearch.http.HttpPipelinedResponse; import org.opensearch.http.HttpResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java index cda66b8d828fa..adf4d59a0c139 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -41,7 +41,7 @@ import io.netty.channel.SimpleChannelInboundHandler; import io.netty.handler.codec.http.FullHttpResponse; import io.netty.util.ReferenceCounted; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -53,7 +53,7 @@ import org.opensearch.http.HttpServerTransport; import org.opensearch.http.NullDispatcher; import org.opensearch.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java index eb96f14f10c70..5a43057b1b7d1 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java @@ -58,7 +58,7 @@ import io.netty.handler.codec.http.HttpVersion; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; @@ -98,8 +98,8 @@ import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.OK; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java index cd263a20c9a74..f80d7f41b5f55 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -33,7 +33,7 @@ package org.opensearch.transport.netty4; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java index 6aafd78ab2f2c..d3fa8ea56ffe7 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java @@ -38,8 +38,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.bytes.AbstractBytesReferenceTestCase; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java index 78a3a353fbf2f..5d7841df4bf33 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java @@ -33,7 +33,7 @@ import org.opensearch.Version; import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.network.NetworkUtils; import org.opensearch.common.settings.Settings; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java index 2546e2fba8c58..619f473b8bef2 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; diff --git a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java index 331316b3757a9..7725eda7f1cde 100644 --- a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -44,7 +44,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/plugins/analysis-icu/src/main/java/org/opensearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/opensearch/plugin/analysis/icu/AnalysisICUPlugin.java index 1f9d7d7463fbd..686ea4fba10f6 100644 --- a/plugins/analysis-icu/src/main/java/org/opensearch/plugin/analysis/icu/AnalysisICUPlugin.java +++ b/plugins/analysis-icu/src/main/java/org/opensearch/plugin/analysis/icu/AnalysisICUPlugin.java @@ -35,7 +35,7 @@ import static java.util.Collections.singletonMap; import org.apache.lucene.analysis.Analyzer; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.index.analysis.AnalyzerProvider; import org.opensearch.index.analysis.CharFilterFactory; import org.opensearch.index.analysis.IcuAnalyzerProvider; diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IcuTokenizerFactoryTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IcuTokenizerFactoryTests.java index 115171aec184f..25ff505cddab8 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IcuTokenizerFactoryTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IcuTokenizerFactoryTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.icu.AnalysisICUPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuAnalysisTests.java index b7f42d524dbe8..2a5268d0c0d9d 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuAnalysisTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuAnalysisTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.analysis; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.icu.AnalysisICUPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuCollationTokenFilterTests.java index a7bd2ca5a0440..980fcc6deb837 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuCollationTokenFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuCollationTokenFilterTests.java @@ -40,7 +40,7 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.icu.AnalysisICUPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java index 4ba8a6ca0b1f1..9bfa997badaad 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java @@ -35,7 +35,7 @@ import com.ibm.icu.text.Normalizer2; import org.apache.lucene.analysis.CharFilter; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.icu.AnalysisICUPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java index 03d9df6ebd6b2..62386c2bd2e3d 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-phonetic/src/test/java/org/opensearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/opensearch/index/analysis/SimplePhoneticAnalysisTests.java index 69d9fe9762c32..b8a28e4b6f36d 100644 --- a/plugins/analysis-phonetic/src/test/java/org/opensearch/index/analysis/SimplePhoneticAnalysisTests.java +++ b/plugins/analysis-phonetic/src/test/java/org/opensearch/index/analysis/SimplePhoneticAnalysisTests.java @@ -39,7 +39,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.AnalysisPhoneticPlugin; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.MatcherAssert; diff --git a/plugins/analysis-smartcn/src/test/java/org/opensearch/index/analysis/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/opensearch/index/analysis/SimpleSmartChineseAnalysisTests.java index 1c71490dd9294..1e99d5b624da6 100644 --- a/plugins/analysis-smartcn/src/test/java/org/opensearch/index/analysis/SimpleSmartChineseAnalysisTests.java +++ b/plugins/analysis-smartcn/src/test/java/org/opensearch/index/analysis/SimpleSmartChineseAnalysisTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.analysis; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.MatcherAssert; diff --git a/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/PolishAnalysisTests.java index 56e48c4fb30b0..ee9b2a6520de2 100644 --- a/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/PolishAnalysisTests.java +++ b/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/PolishAnalysisTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.pl.PolishAnalyzer; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.analysis.pl.PolishStemTokenFilterFactory; import org.opensearch.plugin.analysis.stempel.AnalysisStempelPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/SimplePolishTokenFilterTests.java index fb47a6a24c198..34906b6508856 100644 --- a/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/SimplePolishTokenFilterTests.java +++ b/plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/SimplePolishTokenFilterTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.stempel.AnalysisStempelPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/SimpleUkrainianAnalyzerTests.java b/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/SimpleUkrainianAnalyzerTests.java index deb5cf275d7f6..1e9cde8569063 100644 --- a/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/SimpleUkrainianAnalyzerTests.java +++ b/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/SimpleUkrainianAnalyzerTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/UkrainianAnalysisTests.java b/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/UkrainianAnalysisTests.java index d0aefdaa1d9ec..5c2528b4814bd 100644 --- a/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/UkrainianAnalysisTests.java +++ b/plugins/analysis-ukrainian/src/test/java/org/opensearch/index/analysis/UkrainianAnalysisTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.MatcherAssert; diff --git a/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/opensearch/discovery/ec2/AmazonEC2Fixture.java b/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/opensearch/discovery/ec2/AmazonEC2Fixture.java index 21ff9ef0e104b..6aba3c8fa9241 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/opensearch/discovery/ec2/AmazonEC2Fixture.java +++ b/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/opensearch/discovery/ec2/AmazonEC2Fixture.java @@ -40,7 +40,7 @@ import org.opensearch.common.Booleans; import org.opensearch.common.SuppressForbidden; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.fixture.AbstractHttpFixture; import javax.xml.XMLConstants; diff --git a/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2ClientSettings.java index 8f09276e6a627..14d39cf762bed 100644 --- a/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2ClientSettings.java +++ b/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2ClientSettings.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2NetworkTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2NetworkTests.java index c8cb26026228b..47846cada48f6 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2NetworkTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2NetworkTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.junit.AfterClass; import org.junit.Before; diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java index ebb2fb6dd1783..cc65e92b1485a 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java @@ -39,7 +39,7 @@ import org.opensearch.Version; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.Streams; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.PageCacheRecycler; diff --git a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/opensearch/cloud/gce/GCEFixture.java b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/opensearch/cloud/gce/GCEFixture.java index f1eba6a635504..f588316fdc32f 100644 --- a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/opensearch/cloud/gce/GCEFixture.java +++ b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/opensearch/cloud/gce/GCEFixture.java @@ -37,7 +37,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.path.PathTrie; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import org.opensearch.test.fixture.AbstractHttpFixture; diff --git a/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java b/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java index 6eac0e59fa2ed..86ccd925551fd 100644 --- a/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java +++ b/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java @@ -26,7 +26,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginInfo; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/CorrelationVectorsEngineIT.java b/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/CorrelationVectorsEngineIT.java index f8736f05dadfc..dd83dfa84dbdb 100644 --- a/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/CorrelationVectorsEngineIT.java +++ b/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/CorrelationVectorsEngineIT.java @@ -22,9 +22,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; -import org.opensearch.rest.RestStatus; import org.opensearch.test.rest.OpenSearchRestTestCase; import java.io.IOException; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java index 6945f21a0fd7c..ea05302733ff1 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java @@ -14,7 +14,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContext.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContext.java index c7a23dabfd312..fef9200a73091 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContext.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContext.java @@ -9,9 +9,9 @@ package org.opensearch.plugin.correlation.core.index; import org.apache.lucene.index.VectorSimilarityFunction; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java index a71a7874f178e..f43f8d0e63755 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java @@ -11,9 +11,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleRequest.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleRequest.java index 1d1df00d113c0..3fe25d144059d 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleRequest.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.plugin.correlation.rules.model.CorrelationRule; import org.opensearch.rest.RestRequest; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleResponse.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleResponse.java index 16ac675a5f5db..8ba975a6b9b35 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleResponse.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/action/IndexCorrelationRuleResponse.java @@ -9,13 +9,13 @@ package org.opensearch.plugin.correlation.rules.action; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugin.correlation.rules.model.CorrelationRule; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationQuery.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationQuery.java index 56be3622ea49f..0c90e08da9cbc 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationQuery.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationQuery.java @@ -10,9 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java index 042554c196a74..56cb198484f56 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java @@ -10,9 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java index 72d611e515140..ee29352d3e0f0 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java @@ -23,7 +23,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestResponseListener; import java.io.IOException; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/transport/TransportIndexCorrelationRuleAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/transport/TransportIndexCorrelationRuleAction.java index eb7a9a7568442..5fb31f1f352cb 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/transport/TransportIndexCorrelationRuleAction.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/transport/TransportIndexCorrelationRuleAction.java @@ -32,7 +32,7 @@ import org.opensearch.plugin.correlation.utils.CorrelationRuleIndices; import org.opensearch.plugin.correlation.utils.IndexUtils; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContextTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContextTests.java index 7b9d4e31d7e3a..1e61cd1de549d 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContextTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/CorrelationParamsContextTests.java @@ -10,10 +10,10 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.Assert; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/VectorFieldTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/VectorFieldTests.java index 00cec228de720..861870ab02aa1 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/VectorFieldTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/VectorFieldTests.java @@ -13,8 +13,8 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.common.Randomness; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java index 82c2cc1a07aaf..19dff08b63d5f 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java @@ -17,11 +17,11 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Explicit; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ContentPath; diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java index fd3c7220aad74..f23a4f25302b1 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java @@ -14,16 +14,16 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; diff --git a/plugins/examples/custom-settings/src/main/java/org/opensearch/example/customsettings/ExampleCustomSettingsConfig.java b/plugins/examples/custom-settings/src/main/java/org/opensearch/example/customsettings/ExampleCustomSettingsConfig.java index cb2e28210faf1..19efdc0982ddb 100644 --- a/plugins/examples/custom-settings/src/main/java/org/opensearch/example/customsettings/ExampleCustomSettingsConfig.java +++ b/plugins/examples/custom-settings/src/main/java/org/opensearch/example/customsettings/ExampleCustomSettingsConfig.java @@ -33,7 +33,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; diff --git a/plugins/examples/custom-significance-heuristic/src/main/java/org/opensearch/example/customsigheuristic/SimpleHeuristic.java b/plugins/examples/custom-significance-heuristic/src/main/java/org/opensearch/example/customsigheuristic/SimpleHeuristic.java index cd51833d91dfd..0caf287d460c5 100644 --- a/plugins/examples/custom-significance-heuristic/src/main/java/org/opensearch/example/customsigheuristic/SimpleHeuristic.java +++ b/plugins/examples/custom-significance-heuristic/src/main/java/org/opensearch/example/customsigheuristic/SimpleHeuristic.java @@ -32,8 +32,8 @@ package org.opensearch.example.customsigheuristic; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; diff --git a/plugins/examples/custom-significance-heuristic/src/test/java/org/opensearch/example/customsigheuristic/SimpleHeuristicWireTests.java b/plugins/examples/custom-significance-heuristic/src/test/java/org/opensearch/example/customsigheuristic/SimpleHeuristicWireTests.java index 68d6aa9e52a30..fb93fc3bc49c8 100644 --- a/plugins/examples/custom-significance-heuristic/src/test/java/org/opensearch/example/customsigheuristic/SimpleHeuristicWireTests.java +++ b/plugins/examples/custom-significance-heuristic/src/test/java/org/opensearch/example/customsigheuristic/SimpleHeuristicWireTests.java @@ -32,7 +32,7 @@ package org.opensearch.example.customsigheuristic; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggester.java b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggester.java index 05f26a8e401e1..5ba45a045489a 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggester.java +++ b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggester.java @@ -34,7 +34,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.CharsRefBuilder; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.Suggester; diff --git a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java index 3ba67fe1b9f0a..9255d3a2f299e 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java +++ b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java @@ -33,9 +33,9 @@ package org.opensearch.example.customsuggester; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestionBuilder.java b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestionBuilder.java index 922ca4f049740..733568590037a 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestionBuilder.java +++ b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestionBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.example.customsuggester; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/plugins/examples/rescore/src/main/java/org/opensearch/example/rescore/ExampleRescoreBuilder.java b/plugins/examples/rescore/src/main/java/org/opensearch/example/rescore/ExampleRescoreBuilder.java index a016f7ba10f33..f0a1fc9511de6 100644 --- a/plugins/examples/rescore/src/main/java/org/opensearch/example/rescore/ExampleRescoreBuilder.java +++ b/plugins/examples/rescore/src/main/java/org/opensearch/example/rescore/ExampleRescoreBuilder.java @@ -39,8 +39,8 @@ import org.apache.lucene.search.TopDocs; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/plugins/examples/rescore/src/test/java/org/opensearch/example/rescore/ExampleRescoreBuilderTests.java b/plugins/examples/rescore/src/test/java/org/opensearch/example/rescore/ExampleRescoreBuilderTests.java index 53bd89999f990..88f845d86f4aa 100644 --- a/plugins/examples/rescore/src/test/java/org/opensearch/example/rescore/ExampleRescoreBuilderTests.java +++ b/plugins/examples/rescore/src/test/java/org/opensearch/example/rescore/ExampleRescoreBuilderTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.search.rescore.RescoreContext; import org.opensearch.test.AbstractWireSerializingTestCase; import org.opensearch.test.OpenSearchTestCase; diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java index 4e4648a87fbfc..834c8a448d3d5 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java @@ -35,7 +35,7 @@ import java.util.Collection; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 4b11f2e3305e6..67e79addfedc5 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -49,7 +49,7 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.blobstore.OpenSearchMockAPIBasedRepositoryIntegTestCase; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java index f7c8484a887ae..1bc0e955715f1 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java @@ -35,7 +35,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.AffixSetting; import org.opensearch.common.settings.Setting.Property; diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java index 45384a16b0c0b..8b68ccebf8c53 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java @@ -45,7 +45,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; @@ -56,7 +56,7 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 982dd0e566014..f0f0fb7681c1a 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -50,7 +50,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.Streams; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.MockSecureSettings; diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageClientSettings.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageClientSettings.java index d31cac55ffd64..4325708596870 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageClientSettings.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageClientSettings.java @@ -35,7 +35,7 @@ import com.google.auth.oauth2.ServiceAccountCredentials; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 4800394e428b9..488376d36cdc4 100644 --- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -44,8 +44,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; @@ -59,7 +59,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.repositories.blobstore.AbstractBlobContainerRetriesTestCase; import org.opensearch.repositories.blobstore.OpenSearchMockAPIBasedRepositoryIntegTestCase; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import org.threeten.bp.Duration; diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java index 1c89c0df962ee..fc6cce9b8ae17 100644 --- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -36,7 +36,7 @@ import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.storage.Storage; import org.hamcrest.Matchers; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index 66677b0327191..0ad61906a2104 100644 --- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.test.OpenSearchTestCase; import javax.security.auth.Subject; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 25e9797018102..8097629ed0773 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -37,7 +37,7 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index 954b79035429f..07abb69c11bdd 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -45,7 +45,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 57d0387c96095..045ce73daf5a3 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.hash.MessageDigests; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ByteBufUtils.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ByteBufUtils.java index db9ba4edd644f..f0129e4760c60 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ByteBufUtils.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ByteBufUtils.java @@ -36,8 +36,8 @@ import io.netty.buffer.Unpooled; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java index 73c603ffc059e..5007fc9076ca7 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java @@ -44,10 +44,10 @@ import io.netty.handler.codec.http.cookie.ServerCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.http.HttpRequest; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.AbstractMap; import java.util.Collection; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpResponse.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpResponse.java index 4284b0e2948bc..0d3ce72c6646b 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpResponse.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpResponse.java @@ -37,9 +37,9 @@ import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpVersion; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.http.HttpResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; public class NioHttpResponse extends DefaultFullHttpResponse implements HttpResponse { diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTcpChannel.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTcpChannel.java index 81a07fc646907..48b797840f5ff 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTcpChannel.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTcpChannel.java @@ -33,7 +33,7 @@ package org.opensearch.transport.nio; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.nio.NioSocketChannel; import org.opensearch.transport.TcpChannel; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java index b12247c9c711c..1509e0b179bfe 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java @@ -40,7 +40,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java index a27512fbec7c2..67598aec154fa 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java index a769ab815e0ea..29ef19a2aec87 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java @@ -33,8 +33,8 @@ package org.opensearch.transport.nio; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.io.IOUtils; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java index 072664d992b78..c606a4818a324 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java @@ -47,7 +47,7 @@ import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.HttpVersion; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; @@ -63,7 +63,7 @@ import org.opensearch.nio.SocketChannelContext; import org.opensearch.nio.TaskScheduler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java index 70da3496c2534..87b2234664909 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java @@ -43,13 +43,13 @@ import io.netty.handler.codec.http.QueryStringDecoder; import org.opensearch.common.Randomness; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.http.HttpPipelinedRequest; import org.opensearch.http.HttpPipelinedResponse; import org.opensearch.http.HttpRequest; import org.opensearch.http.HttpResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java index 9106e9608c778..c69fe23002dfe 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java @@ -47,7 +47,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; @@ -88,8 +88,8 @@ import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.OK; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java index b42fb7fc4a749..4b06c4e15bce7 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; diff --git a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java index df130caaa811c..724ac9883efaa 100644 --- a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java @@ -37,10 +37,10 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/qa/logging-config/src/test/java/org/opensearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/opensearch/common/logging/JsonLoggerTests.java index c760677aa3cd0..7fbfd6929ebdf 100644 --- a/qa/logging-config/src/test/java/org/opensearch/common/logging/JsonLoggerTests.java +++ b/qa/logging-config/src/test/java/org/opensearch/common/logging/JsonLoggerTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.env.Environment; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.FeatureMatcher; diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java index f1f469544f634..f9810e027bb1e 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java @@ -48,7 +48,7 @@ import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.opensearch.test.rest.yaml.ObjectPath; diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/SearchingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/SearchingIT.java index 6166d6f3b086e..1f55a40bbc868 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/SearchingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/SearchingIT.java @@ -9,41 +9,23 @@ package org.opensearch.backwards; import org.apache.hc.core5.http.HttpHost; -import org.opensearch.LegacyESVersion; -import org.opensearch.Version; import org.opensearch.action.get.MultiGetRequest; import org.opensearch.action.get.MultiGetResponse; -import org.opensearch.backwards.IndexingIT.Nodes; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; -import org.opensearch.client.ResponseException; import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.Strings; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.common.xcontent.support.XContentMapValues; -import org.opensearch.index.seqno.SeqNoStats; -import org.opensearch.rest.RestStatus; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.opensearch.test.rest.yaml.ObjectPath; import java.io.IOException; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; public class SearchingIT extends OpenSearchRestTestCase { public void testMultiGet() throws Exception { diff --git a/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java index e54a3a1221fc9..5f52250c7fb0c 100644 --- a/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java @@ -54,7 +54,7 @@ import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.CreateIndexResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java index 0902f735b6448..3dd9f371f06fd 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java @@ -47,7 +47,7 @@ import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.rest.yaml.ObjectPath; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.hamcrest.Matchers; diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/AwarenessAttributeDecommissionRestIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/AwarenessAttributeDecommissionRestIT.java index b7228a75984fa..61afb3ffdcf98 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/AwarenessAttributeDecommissionRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/AwarenessAttributeDecommissionRestIT.java @@ -15,7 +15,7 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.routing.WeightedRouting; import org.opensearch.common.settings.Settings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/DanglingIndicesRestIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/DanglingIndicesRestIT.java index 8ce546e327c31..42c7fd667fd8f 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/DanglingIndicesRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/DanglingIndicesRestIT.java @@ -54,8 +54,8 @@ import static org.opensearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES; import static org.opensearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING; import static org.opensearch.indices.IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING; -import static org.opensearch.rest.RestStatus.ACCEPTED; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.ACCEPTED; +import static org.opensearch.core.rest.RestStatus.OK; import static org.opensearch.test.XContentTestUtils.createJsonMapView; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java index 45202f6dcbfce..78398e10b9ce8 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java @@ -20,7 +20,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.identity.shiro.ShiroIdentityPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchTestCase; diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/IndexingPressureRestIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/IndexingPressureRestIT.java index bac4304db1ef6..6ff2dee2812e9 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/IndexingPressureRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/IndexingPressureRestIT.java @@ -46,9 +46,9 @@ import java.util.ArrayList; import java.util.Map; -import static org.opensearch.rest.RestStatus.CREATED; -import static org.opensearch.rest.RestStatus.OK; -import static org.opensearch.rest.RestStatus.TOO_MANY_REQUESTS; +import static org.opensearch.core.rest.RestStatus.CREATED; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.TOO_MANY_REQUESTS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java index c018bc68e051a..9e663ac042d91 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/ShardIndexingPressureRestIT.java @@ -19,9 +19,9 @@ import java.util.ArrayList; import java.util.Map; -import static org.opensearch.rest.RestStatus.CREATED; -import static org.opensearch.rest.RestStatus.OK; -import static org.opensearch.rest.RestStatus.TOO_MANY_REQUESTS; +import static org.opensearch.core.rest.RestStatus.CREATED; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.TOO_MANY_REQUESTS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/TestResponseHeaderRestAction.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/TestResponseHeaderRestAction.java index c95b4f0070a80..6cf92626e335a 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/TestResponseHeaderRestAction.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/TestResponseHeaderRestAction.java @@ -36,7 +36,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.List; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java index 02b4f0a216ac7..5054039319392 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java @@ -94,7 +94,7 @@ import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.Requests; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.query.QueryBuilders; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java index 56112ebdbc999..e095927efdd01 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.env.Environment; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java index 08b43afc6e8e3..4cbcc5b9bb507 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java @@ -58,7 +58,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.InternalTestCluster; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index 6743efda8e5e3..161dee1a96e3c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -51,8 +51,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.SetOnce; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.set.Sets; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 474a5069ad091..13200ba2ca43e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.settings.Settings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java index 63141b590a67f..daf4769e9fabb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java index 3579539f3998a..51598d7775623 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java @@ -68,7 +68,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.engine.SegmentsStats; import org.opensearch.index.query.TermsQueryBuilder; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/SplitIndexIT.java index 89e11984621da..ea53efe176eaf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/SplitIndexIT.java @@ -62,7 +62,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.engine.SegmentsStats; import org.opensearch.index.query.TermsQueryBuilder; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamRolloverIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamRolloverIT.java index bac16dd4b7cc0..6afa48da0918e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamRolloverIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamRolloverIT.java @@ -10,7 +10,7 @@ import org.opensearch.action.admin.indices.rollover.RolloverResponse; import org.opensearch.cluster.metadata.DataStream; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Collections; import java.util.stream.Collectors; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java index 6a9394d6582ac..44f66dd4e0f90 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java @@ -16,7 +16,7 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.DataStream; import org.opensearch.cluster.metadata.Template; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamUsageIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamUsageIT.java index 46f23e40f0864..2d06b594a598c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamUsageIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamUsageIT.java @@ -15,7 +15,7 @@ import org.opensearch.cluster.metadata.DataStream; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.Arrays; import java.util.List; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeIT.java index 195817bf04cc9..09af533292e9a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeIT.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java index fe1bc05dc5f20..ee516a53fadef 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java @@ -34,7 +34,7 @@ import org.opensearch.action.RequestValidators; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index c8501171eb5da..52b2ea93fe071 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java index ec8d02a8c57b7..1c2e8200abb6d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java @@ -45,12 +45,12 @@ import org.opensearch.action.support.replication.ReplicationRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ingest.IngestTestPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java index 687a4e9b733fd..d0285f24e22fe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java @@ -37,7 +37,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.transport.RemoteTransportException; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java b/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java index 86bca417308a3..58c18be59fdac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java @@ -39,9 +39,9 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; import org.opensearch.env.Environment; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java index 8cd7bdeb0c822..c22319271b4cf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java @@ -48,8 +48,8 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AtomicArray; @@ -63,7 +63,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHit; import org.opensearch.search.aggregations.AbstractAggregationBuilder; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/support/WaitActiveShardCountIT.java b/server/src/internalClusterTest/java/org/opensearch/action/support/WaitActiveShardCountIT.java index e919b2b85e079..c82af8bce6e2d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/support/WaitActiveShardCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/support/WaitActiveShardCountIT.java @@ -40,7 +40,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import static org.opensearch.common.unit.TimeValue.timeValueMillis; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index 725cc894185e3..9f60e65eca297 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -41,12 +41,12 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.NetworkPlugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java index 6009ad8382e54..e5a46c3c6bb9f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java @@ -55,17 +55,17 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; import org.opensearch.gateway.GatewayService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfoTests; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java index 313f9e0da17aa..7f454ce848f6b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java @@ -52,7 +52,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java index d2c4859e35b1d..61171b0a817b0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java @@ -46,9 +46,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java index e9260e36e50d3..298aff4a5c853 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java @@ -63,8 +63,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java index 0da039d29ba0b..9c8cb9f15e79c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java @@ -56,7 +56,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.discovery.Discovery; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceIT.java index 17d024dfb999d..dd0faa0ea8f61 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceIT.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/AllocationIdIT.java index a57ea705f4f88..3f0243136d790 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/AllocationIdIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/AllocationIdIT.java @@ -50,7 +50,7 @@ import org.opensearch.index.MockEngineFactoryPlugin; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.RemoveCorruptedShardDataCommandIT; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java index d2cce1318772a..017f6fc29ef87 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java @@ -58,7 +58,7 @@ import org.opensearch.index.engine.EngineTestCase; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/WeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/WeightedRoutingIT.java index a7597912c4110..c9456b2418610 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/WeightedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/WeightedRoutingIT.java @@ -16,7 +16,7 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.mockstore.MockRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.disruption.NetworkDisruption; diff --git a/server/src/internalClusterTest/java/org/opensearch/common/settings/ConsistentSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/common/settings/ConsistentSettingsIT.java index 0ffddf815c40b..f8f56cf05fbea 100644 --- a/server/src/internalClusterTest/java/org/opensearch/common/settings/ConsistentSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/common/settings/ConsistentSettingsIT.java @@ -34,6 +34,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Setting.AffixSetting; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/explain/ExplainActionIT.java b/server/src/internalClusterTest/java/org/opensearch/explain/ExplainActionIT.java index 53da0309aa602..746b010c0f4fd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/explain/ExplainActionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/explain/ExplainActionIT.java @@ -35,8 +35,8 @@ import org.apache.lucene.search.Explanation; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.explain.ExplainResponse; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; import org.opensearch.index.query.QueryBuilders; diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/MetadataNodesIT.java index cfd1dd7f9a008..5e30c1ad9c08e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/MetadataNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/MetadataNodesIT.java @@ -39,7 +39,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.discovery.Discovery; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java index 298ec5a8efc10..0b3a689e81b94 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java @@ -50,13 +50,13 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergePolicyConfig; import org.opensearch.index.engine.Engine; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.RecoveryState; diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java index 6d05ecd0b56b0..9a465c2f9121c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java @@ -41,7 +41,7 @@ import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.ReplicationTracker; diff --git a/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java index 2f811d4a901bf..997e8e9d5258b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/get/GetActionIT.java @@ -43,17 +43,17 @@ import org.opensearch.action.get.MultiGetRequestBuilder; import org.opensearch.action.get.MultiGetResponse; import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.InternalSettingsPlugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/opensearch/index/FinalPipelineIT.java index 32b2097272518..22dc1224f6e09 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/FinalPipelineIT.java @@ -45,9 +45,9 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; @@ -60,7 +60,7 @@ import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.ScriptService; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/IndexRequestBuilderIT.java b/server/src/internalClusterTest/java/org/opensearch/index/IndexRequestBuilderIT.java index 9432f28a0a59e..925a1b50fd6a8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/IndexRequestBuilderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/IndexRequestBuilderIT.java @@ -34,8 +34,8 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.QueryBuilders; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java b/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java index f8ed719e0f292..60ff82e617dbd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java @@ -21,7 +21,7 @@ import org.opensearch.indices.replication.SegmentReplicationBaseIT; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java index e8cc8543d36e9..43d8173103656 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java @@ -23,7 +23,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java index e0fc1e93df686..707a0f6ab8818 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java @@ -25,7 +25,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/WaitUntilRefreshIT.java b/server/src/internalClusterTest/java/org/opensearch/index/WaitUntilRefreshIT.java index e38b128c04fde..0927d274b1265 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/WaitUntilRefreshIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/WaitUntilRefreshIT.java @@ -44,7 +44,7 @@ import org.opensearch.client.Requests; import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java index b548d4061475c..2a47e6ce74e58 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java @@ -42,7 +42,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java b/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java index ed6074b39c8a7..e12c5d44df69b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java @@ -44,7 +44,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 1524acc4e7dd7..ba9f335cd24d4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -55,7 +55,7 @@ import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; @@ -63,10 +63,11 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java index 28f7db77738ba..b6124ff09d992 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -67,11 +67,12 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; import org.opensearch.gateway.GatewayMetaState; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergePolicyConfig; import org.opensearch.index.MockEngineFactoryPlugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java index 960e17b76acb5..28753e495881f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java @@ -62,7 +62,7 @@ import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.Lucene; @@ -71,13 +71,13 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergePolicyConfig; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.monitor.fs.FsInfo; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesLifecycleListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesLifecycleListenerIT.java index 1bd616b461aff..e427fb8749e64 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesLifecycleListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesLifecycleListenerIT.java @@ -44,11 +44,11 @@ import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 2dc6b2085b866..2ab44f8318617 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -56,7 +56,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.indices.breaker.CircuitBreakerStats; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/DanglingIndicesIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/DanglingIndicesIT.java index ede18af80f6ca..79e1df656484e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/DanglingIndicesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/DanglingIndicesIT.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.indices.IndicesService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.InternalTestCluster; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java index 72b9b32236371..c31b5e1f3bc5b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java @@ -84,7 +84,7 @@ import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentType; import org.opensearch.gateway.ReplicaShardAllocatorIT; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.MockEngineFactoryPlugin; @@ -96,7 +96,7 @@ import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreStats; import org.opensearch.indices.IndicesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 4692210ccc577..64c6ebbb33482 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -20,14 +20,14 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java index c1326c1b50c9e..a82fd8d845709 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java @@ -12,7 +12,7 @@ import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 099c15267c2f7..d345578c7de6e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -58,7 +58,7 @@ import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.NRTReplicationReaderManager; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.search.SearchService; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java index cdebe6c869be5..d6dce78061a7a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java @@ -48,7 +48,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.PeerRecoverySourceService; import org.opensearch.indices.recovery.StartRecoveryRequest; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java index ca1e1399f8fdc..fde30f35d1b6d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java @@ -49,7 +49,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java index be8a1ad4f0c61..35f2b99c94625 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java @@ -49,13 +49,14 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchType; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; @@ -1319,7 +1320,7 @@ public void testFilterCacheStats() throws Exception { * Test that we can safely concurrently index and get stats. This test was inspired by a serialization issue that arose due to a race * getting doc stats during heavy indexing. The race could lead to deleted docs being negative which would then be serialized as a * variable-length long. Since serialization of negative longs using a variable-length format was unsupported - * ({@link org.opensearch.common.io.stream.StreamOutput#writeVLong(long)}), the stream would become corrupted. Here, we want to test + * ({@link StreamOutput#writeVLong(long)}), the stream would become corrupted. Here, we want to test * that we can continue to get stats while indexing. */ public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierException, InterruptedException, ExecutionException { diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/store/IndicesStoreIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/store/IndicesStoreIntegrationIT.java index a532e16f80d19..957aa7fd05a13 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/store/IndicesStoreIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/store/IndicesStoreIntegrationIT.java @@ -54,8 +54,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java index b35a5a0c277f7..a6381b4450010 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java index 522938003838b..49d09cada59b3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java @@ -51,7 +51,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Requests; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java index a615cceffb5df..4b4a0d9d0157c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.node.NodeService; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java b/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java index c9d18e64ca038..7109d8e331ce3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java @@ -39,7 +39,7 @@ import org.opensearch.action.get.MultiGetRequestBuilder; import org.opensearch.action.get.MultiGetResponse; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.fetch.subphase.FetchSourceContext; diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java index 26b3e9ae336dc..a675eb7c77344 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java @@ -50,7 +50,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.DocsStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.plugins.Plugin; import org.opensearch.search.sort.SortOrder; diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java index c691d36f2d7e2..b3821c7896b8e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java @@ -64,7 +64,7 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index 64d5f06f061a9..3fe7f3d553a1b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -11,11 +11,11 @@ import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java index c09bd743ca361..4e7f2ae486c93 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java @@ -35,7 +35,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.fs.FsBlobStore; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; diff --git a/server/src/internalClusterTest/java/org/opensearch/routing/SimpleRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/routing/SimpleRoutingIT.java index 8909b9deece9b..f43ddfcdfc47b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/routing/SimpleRoutingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/routing/SimpleRoutingIT.java @@ -55,8 +55,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import static org.hamcrest.Matchers.containsString; diff --git a/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java index b4823bb482bfa..f6e6b7994db65 100644 --- a/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java @@ -15,7 +15,7 @@ import org.opensearch.index.mapper.MockFieldFilterPlugin; import org.opensearch.node.NodeMocksPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.MockSearchService; import org.opensearch.test.MockHttpTransport; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/script/StoredScriptsIT.java b/server/src/internalClusterTest/java/org/opensearch/script/StoredScriptsIT.java index 0d3a7154559cb..448bbf5e883ec 100644 --- a/server/src/internalClusterTest/java/org/opensearch/script/StoredScriptsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/script/StoredScriptsIT.java @@ -31,7 +31,7 @@ package org.opensearch.script; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java index a0bb2989b8328..28a4db5ecaf9d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java @@ -29,9 +29,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.search.stats.SearchStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.snapshots.mockstore.MockRepository; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java index 9b40df5f19b9f..e554a3be20528 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java index 47b7c07c1ead9..ed3edc8c624f8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java @@ -41,7 +41,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.InnerHitBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.filter.Filter; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java index 2065b122aac87..2034bbb7e13bc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -36,7 +36,7 @@ import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java index ab2460fb970c7..e8e21d3580e1c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java @@ -34,7 +34,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java index 67d90cf671fbb..8fe8876c7593b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java @@ -34,7 +34,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java index aeb137436b7b0..bb7aa9514564a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java @@ -35,7 +35,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java index a63c3287ea124..070a2bd7d6b7c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java @@ -22,8 +22,8 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.plugins.ActionPlugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java index 3c2aa6642633e..2b4aba029466a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.common.settings.Settings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchService; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java index a3d68d3d16122..e9ea88b5945a3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java @@ -38,10 +38,10 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java index 9a3505a6e0abe..d83f1eb776b20 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java @@ -33,7 +33,7 @@ package org.opensearch.search.fetch.subphase; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchQueryBuilder; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 412c6188b8719..f2a22b99a86a3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -65,7 +65,7 @@ import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.plugins.AnalysisPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java index 7945f1f4274fb..90d0a59f7b58d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java @@ -37,8 +37,8 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Numbers; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.document.DocumentField; import org.opensearch.common.settings.Settings; @@ -52,7 +52,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java index a7a14f3b0d889..a1d3d9f22af04 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java @@ -37,8 +37,8 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.common.Priority; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.query.functionscore.DecayFunction; import org.opensearch.index.query.functionscore.DecayFunctionBuilder; import org.opensearch.index.query.functionscore.DecayFunctionParser; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java index 3444ddbca9e65..7f306b3ee550f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Priority; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; import org.opensearch.common.geo.builders.CoordinatesBuilder; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java index e519a74ad8eaf..55a2a1fdde2b5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java @@ -45,7 +45,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.node.Node; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java index 2a5fd929d61c3..0e0f4873297ba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java @@ -73,7 +73,7 @@ import org.opensearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.opensearch.plugins.AnalysisPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.AggregationBuilders; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java index a1ee8017894da..3ad15f852ebc5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java @@ -41,7 +41,7 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Priority; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; @@ -52,7 +52,7 @@ import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index fdbdb7fa94ed1..5d7c6d5891b83 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -46,7 +46,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java index fdac991449e55..83e732b39103e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java @@ -54,7 +54,7 @@ import org.opensearch.index.query.functionscore.ScoreFunctionBuilders; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java index f6432eec9358b..75bae17d867ff 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -42,7 +42,7 @@ import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.suggest.CompletionSuggestSearchIT.CompletionMappingBuilder; import org.opensearch.search.suggest.completion.CompletionSuggestionBuilder; import org.opensearch.search.suggest.completion.context.CategoryContextMapping; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 973ab82c6750e..0d10124b585f5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -60,9 +60,9 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.ParseField; import org.opensearch.common.Priority; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; @@ -74,7 +74,7 @@ import org.opensearch.env.Environment; import org.opensearch.index.seqno.RetentionLeaseActions; import org.opensearch.index.seqno.RetentionLeases; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.node.Node; @@ -84,7 +84,7 @@ import org.opensearch.rest.AbstractRestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.admin.cluster.RestClusterStateAction; import org.opensearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.opensearch.snapshots.mockstore.MockRepository; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index b566a1629abe5..dd6d9834f63da 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -49,7 +49,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryData; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.mockstore.MockRepository; import java.io.IOException; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index e362b7f61e8e6..5649169ee15b7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -56,11 +56,12 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.rest.RestStatus; import org.opensearch.index.IndexSettings; import org.opensearch.indices.InvalidIndexNameException; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.rest.RestStatus; + import org.opensearch.test.InternalTestCluster; import java.io.IOException; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java index bf9ea4a3a781f..300e1db09b4c5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java @@ -31,7 +31,7 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java index 6ab36eaa4d64b..0a47b916a3d60 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -17,11 +17,11 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java index 9736aba92f3d1..4bba25039d376 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -69,13 +69,13 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.util.BytesRefUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineTestCase; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotCustomPluginStateIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotCustomPluginStateIT.java index ea1635b1d8053..85fedead80a85 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotCustomPluginStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotCustomPluginStateIT.java @@ -40,8 +40,8 @@ import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.opensearch.action.ingest.DeletePipelineRequest; import org.opensearch.action.ingest.GetPipelineResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ingest.IngestTestPlugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java index 16b0e8829b1a7..437b6c25ef4b6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java @@ -53,7 +53,7 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/server/src/internalClusterTest/java/org/opensearch/update/UpdateNoopIT.java b/server/src/internalClusterTest/java/org/opensearch/update/UpdateNoopIT.java index 1c5d0c84a9a2f..292cd7f97ca58 100644 --- a/server/src/internalClusterTest/java/org/opensearch/update/UpdateNoopIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/update/UpdateNoopIT.java @@ -34,7 +34,7 @@ import org.opensearch.action.update.UpdateRequestBuilder; import org.opensearch.action.update.UpdateResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java index 86a51e7367ade..30701e3bae7b7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.client.Client; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java index 2194152284d37..ca41a7ddcd76e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -38,14 +38,14 @@ import org.opensearch.cluster.coordination.LinearizabilityChecker; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.AbstractDisruptionTestCase; import org.opensearch.index.engine.VersionConflictEngineException; diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index a2c59de7832d4..963044a3f58d4 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -45,7 +45,7 @@ import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.index.mapper.RangeType; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/OpenSearchGenerationException.java b/server/src/main/java/org/opensearch/OpenSearchGenerationException.java index 23f93263ee3d2..50b021d41123f 100644 --- a/server/src/main/java/org/opensearch/OpenSearchGenerationException.java +++ b/server/src/main/java/org/opensearch/OpenSearchGenerationException.java @@ -32,7 +32,7 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/OpenSearchParseException.java b/server/src/main/java/org/opensearch/OpenSearchParseException.java index 57eab19cc01f4..c2516402b0d30 100644 --- a/server/src/main/java/org/opensearch/OpenSearchParseException.java +++ b/server/src/main/java/org/opensearch/OpenSearchParseException.java @@ -32,8 +32,8 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/OpenSearchSecurityException.java b/server/src/main/java/org/opensearch/OpenSearchSecurityException.java index 6f725573e71ce..f3b2022f03e38 100644 --- a/server/src/main/java/org/opensearch/OpenSearchSecurityException.java +++ b/server/src/main/java/org/opensearch/OpenSearchSecurityException.java @@ -31,8 +31,8 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index 825197c07b977..d53164427debf 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -8,7 +8,7 @@ package org.opensearch; -import org.opensearch.index.snapshots.IndexShardSnapshotException; +import org.opensearch.core.index.snapshots.IndexShardSnapshotException; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandle; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle; diff --git a/server/src/main/java/org/opensearch/OpenSearchStatusException.java b/server/src/main/java/org/opensearch/OpenSearchStatusException.java index b9cc51eaa2a8d..8061057465498 100644 --- a/server/src/main/java/org/opensearch/OpenSearchStatusException.java +++ b/server/src/main/java/org/opensearch/OpenSearchStatusException.java @@ -32,9 +32,9 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/OpenSearchTimeoutException.java b/server/src/main/java/org/opensearch/OpenSearchTimeoutException.java index 6f97d8fc3dde3..3b1d7086d0584 100644 --- a/server/src/main/java/org/opensearch/OpenSearchTimeoutException.java +++ b/server/src/main/java/org/opensearch/OpenSearchTimeoutException.java @@ -32,7 +32,7 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/ResourceAlreadyExistsException.java b/server/src/main/java/org/opensearch/ResourceAlreadyExistsException.java index 2fbbf459ea293..9566244062626 100644 --- a/server/src/main/java/org/opensearch/ResourceAlreadyExistsException.java +++ b/server/src/main/java/org/opensearch/ResourceAlreadyExistsException.java @@ -32,9 +32,9 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/ResourceNotFoundException.java b/server/src/main/java/org/opensearch/ResourceNotFoundException.java index 6282f9f03b4f9..0a89fa3667542 100644 --- a/server/src/main/java/org/opensearch/ResourceNotFoundException.java +++ b/server/src/main/java/org/opensearch/ResourceNotFoundException.java @@ -31,8 +31,8 @@ package org.opensearch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java index af8fde4c9893c..7899324a3301e 100644 --- a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java @@ -32,8 +32,8 @@ package org.opensearch.action; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportException; diff --git a/server/src/main/java/org/opensearch/action/ActionRequest.java b/server/src/main/java/org/opensearch/action/ActionRequest.java index a6879dd98691a..5313a05ad6fae 100644 --- a/server/src/main/java/org/opensearch/action/ActionRequest.java +++ b/server/src/main/java/org/opensearch/action/ActionRequest.java @@ -32,8 +32,8 @@ package org.opensearch.action; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/ActionResponse.java b/server/src/main/java/org/opensearch/action/ActionResponse.java index ab0544365c0b9..fd13971433d8b 100644 --- a/server/src/main/java/org/opensearch/action/ActionResponse.java +++ b/server/src/main/java/org/opensearch/action/ActionResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/ActionType.java b/server/src/main/java/org/opensearch/action/ActionType.java index c22cddd6fad71..3fad07ca6fce1 100644 --- a/server/src/main/java/org/opensearch/action/ActionType.java +++ b/server/src/main/java/org/opensearch/action/ActionType.java @@ -32,8 +32,9 @@ package org.opensearch.action; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.transport.TransportRequestOptions; /** @@ -63,7 +64,7 @@ public String name() { } /** - * Get a reader that can create a new instance of the class from a {@link org.opensearch.common.io.stream.StreamInput} + * Get a reader that can create a new instance of the class from a {@link StreamInput} */ public Writeable.Reader getResponseReader() { return responseReader; diff --git a/server/src/main/java/org/opensearch/action/DocWriteRequest.java b/server/src/main/java/org/opensearch/action/DocWriteRequest.java index 2810452cd583d..65242f4925669 100644 --- a/server/src/main/java/org/opensearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/opensearch/action/DocWriteRequest.java @@ -38,11 +38,11 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.update.UpdateRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.index.VersionType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/action/DocWriteResponse.java b/server/src/main/java/org/opensearch/action/DocWriteResponse.java index a694b91855e41..d1e3647bc3496 100644 --- a/server/src/main/java/org/opensearch/action/DocWriteResponse.java +++ b/server/src/main/java/org/opensearch/action/DocWriteResponse.java @@ -38,18 +38,18 @@ import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -57,7 +57,7 @@ import java.util.Locale; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; diff --git a/server/src/main/java/org/opensearch/action/FailedNodeException.java b/server/src/main/java/org/opensearch/action/FailedNodeException.java index d05171c0b30de..cebcbd38741a0 100644 --- a/server/src/main/java/org/opensearch/action/FailedNodeException.java +++ b/server/src/main/java/org/opensearch/action/FailedNodeException.java @@ -33,8 +33,8 @@ package org.opensearch.action; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/NoShardAvailableActionException.java b/server/src/main/java/org/opensearch/action/NoShardAvailableActionException.java index 8cb6ff5c9de81..d90ddeeba69af 100644 --- a/server/src/main/java/org/opensearch/action/NoShardAvailableActionException.java +++ b/server/src/main/java/org/opensearch/action/NoShardAvailableActionException.java @@ -33,9 +33,9 @@ package org.opensearch.action; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/NoSuchNodeException.java b/server/src/main/java/org/opensearch/action/NoSuchNodeException.java index 3889809b7f186..58fa110973b2c 100644 --- a/server/src/main/java/org/opensearch/action/NoSuchNodeException.java +++ b/server/src/main/java/org/opensearch/action/NoSuchNodeException.java @@ -32,7 +32,7 @@ package org.opensearch.action; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/OriginalIndices.java b/server/src/main/java/org/opensearch/action/OriginalIndices.java index 96103b1eab0e2..1e24c64bc60fc 100644 --- a/server/src/main/java/org/opensearch/action/OriginalIndices.java +++ b/server/src/main/java/org/opensearch/action/OriginalIndices.java @@ -33,8 +33,8 @@ package org.opensearch.action; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/action/PrimaryMissingActionException.java b/server/src/main/java/org/opensearch/action/PrimaryMissingActionException.java index 95d82d1f43c7b..959a2ee655fe1 100644 --- a/server/src/main/java/org/opensearch/action/PrimaryMissingActionException.java +++ b/server/src/main/java/org/opensearch/action/PrimaryMissingActionException.java @@ -33,7 +33,7 @@ package org.opensearch.action; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/RequestValidators.java b/server/src/main/java/org/opensearch/action/RequestValidators.java index d3ec4ba2f0017..5cdf20616afc5 100644 --- a/server/src/main/java/org/opensearch/action/RequestValidators.java +++ b/server/src/main/java/org/opensearch/action/RequestValidators.java @@ -33,7 +33,7 @@ package org.opensearch.action; import org.opensearch.cluster.ClusterState; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Collection; import java.util.Optional; diff --git a/server/src/main/java/org/opensearch/action/RoutingMissingException.java b/server/src/main/java/org/opensearch/action/RoutingMissingException.java index 18b2dfb89b381..2395bfb21bf4b 100644 --- a/server/src/main/java/org/opensearch/action/RoutingMissingException.java +++ b/server/src/main/java/org/opensearch/action/RoutingMissingException.java @@ -33,10 +33,10 @@ package org.opensearch.action; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.mapper.MapperService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/TaskOperationFailure.java b/server/src/main/java/org/opensearch/action/TaskOperationFailure.java index 25e1ab1bb3d6f..46ad8dc796d09 100644 --- a/server/src/main/java/org/opensearch/action/TaskOperationFailure.java +++ b/server/src/main/java/org/opensearch/action/TaskOperationFailure.java @@ -35,14 +35,14 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/TimestampParsingException.java b/server/src/main/java/org/opensearch/action/TimestampParsingException.java index 7cebdccdbef34..25e22a6dfb6c3 100644 --- a/server/src/main/java/org/opensearch/action/TimestampParsingException.java +++ b/server/src/main/java/org/opensearch/action/TimestampParsingException.java @@ -33,8 +33,8 @@ package org.opensearch.action; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/UnavailableShardsException.java b/server/src/main/java/org/opensearch/action/UnavailableShardsException.java index 75fde3828e904..706d9bb38ed13 100644 --- a/server/src/main/java/org/opensearch/action/UnavailableShardsException.java +++ b/server/src/main/java/org/opensearch/action/UnavailableShardsException.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java index 5051e8c14b72b..625aa91e6ea7f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java index 5ce26aba3b395..5987d718a20ba 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.allocation; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java index f718fd7d8142d..a80e5ff0ea7ab 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java @@ -40,12 +40,12 @@ import org.opensearch.cluster.routing.allocation.AllocationDecision; import org.opensearch.cluster.routing.allocation.ShardAllocationDecision; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.time.Instant; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java index a50c3b2d6f100..30d1e617258c5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java @@ -53,7 +53,7 @@ import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.snapshots.SnapshotsInfoService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 2acf947105171..5c48f8adf8240 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsResponse.java index 6da110b3df1c5..3c511afe1cbff 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/AddVotingConfigExclusionsResponse.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.configuration; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java index 5ba91c1f8f239..95c6d2f4afd47 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsResponse.java index 44352ae68cc68..afbf7af81363f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsResponse.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.configuration; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java index ffdb2735ae69f..727458e8c0971 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java @@ -49,7 +49,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java index b65688dcc30f6..83b08626a2fcd 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java @@ -49,7 +49,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.threadpool.ThreadPool; import org.opensearch.threadpool.ThreadPool.Names; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java index 205be54a36c33..79b7381801da6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java index 2ff634966586a..3d0404c25373b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java @@ -9,8 +9,8 @@ package org.opensearch.action.admin.cluster.decommission.awareness.delete; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/TransportDeleteDecommissionStateAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/TransportDeleteDecommissionStateAction.java index 7d8f4bdd8304c..6d52934a8ba3f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/TransportDeleteDecommissionStateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/TransportDeleteDecommissionStateAction.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java index 773c12c525817..4fe26df19a289 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java index 443e2306be826..41344abf7b153 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java @@ -11,8 +11,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.action.ActionResponse; import org.opensearch.cluster.decommission.DecommissionStatus; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -21,7 +21,7 @@ import java.util.Locale; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Response for decommission status diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/TransportGetDecommissionStateAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/TransportGetDecommissionStateAction.java index d811ab8cf6948..64744a24b4afd 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/TransportGetDecommissionStateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/TransportGetDecommissionStateAction.java @@ -18,7 +18,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java index df364b05dd494..cbf201d4ffeec 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java @@ -11,8 +11,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.decommission.DecommissionAttribute; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java index 163c117a0edd9..13c1775b005b3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java @@ -9,8 +9,8 @@ package org.opensearch.action.admin.cluster.decommission.awareness.put; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/TransportDecommissionAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/TransportDecommissionAction.java index 6f4e3cf82d2ce..f8a145c9063b9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/TransportDecommissionAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/TransportDecommissionAction.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java index b1ebd63ef4676..e595c2560a07b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -40,8 +40,8 @@ import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.Priority; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java index af8fdce81dad1..714abe86df71e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java @@ -40,8 +40,8 @@ import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.health.ClusterStateHealth; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.StatusToXContentObject; @@ -51,7 +51,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java index 606b74b9b85df..d84179fa6bc0a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -58,7 +58,7 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java index 7de148b0a1000..9c33d023f832d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java index 34d0d812ae609..29a1c3d243c9d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.node.hotthreads; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java index a5789cee8b740..5af9ce50a4bfe 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java index 4c71993251f4f..482d9526fa53c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java @@ -38,8 +38,8 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.monitor.jvm.HotThreads; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java index d7fcfa992800e..acf40e3a9de3c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.http.HttpInfo; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java index a078199adb64a..31cacda7c3487 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.node.info; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java index 82f7d58c0ba25..3dcb30a091a94 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/PluginsAndModules.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/PluginsAndModules.java index ccaa089b65517..961fa6dac810f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/PluginsAndModules.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/PluginsAndModules.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.node.info; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; import org.opensearch.plugins.PluginInfo; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index 0bfe24f5b47df..2c4f8522a5a5c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.node.NodeService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessRequest.java index fc67ccb0f5574..e4d9267123862 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessRequest.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessResponse.java index 8fdbf66f10e64..95b4d0d918578 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/liveness/LivenessResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index b5298b5f5eefb..dbfe8378ef394 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -33,16 +33,16 @@ package org.opensearch.action.admin.cluster.node.reload; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import org.opensearch.common.CharArrays; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.settings.SecureString; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java index 36a4a22460126..2f2162947aeea 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.support.nodes.NodesOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; /** * Builder for the reload secure settings nodes request diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java index f6d9a5c7b6453..22044f0c69c48 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index 920c66bc5c543..f0582daee7261 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -43,10 +43,10 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.KeyStoreWrapper; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.plugins.PluginsService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index 4cdc54f9c7952..3b6c85ee6e091 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.discovery.DiscoveryStats; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index f37a837c6f0ef..34cf03812d233 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Arrays; import java.util.HashSet; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java index b24eb44e32e5d..0037d1ea27873 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java @@ -36,8 +36,8 @@ import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 660142f05bab2..615abbaef845d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.node.NodeService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java index 794c942a4e7d2..360765e8f4803 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.node.tasks.cancel; import org.opensearch.action.support.tasks.BaseTasksRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java index b69a473dbbaff..4741504ddd035 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java @@ -36,7 +36,7 @@ import org.opensearch.action.TaskOperationFailure; import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java index 663fe51f35adc..b504c7e6a39b0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java index db8f664580ec4..b32e59fc77794 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java index decd26e3058f5..070b93c788ef0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.node.tasks.list; import org.opensearch.action.support.tasks.BaseTasksRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java index ae9b9d3da6fb2..85b165df68cfa 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java @@ -41,8 +41,8 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodeUsage.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodeUsage.java index d7286b750b934..72599f2910c7f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodeUsage.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodeUsage.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java index 1badfa6d02f15..e31b88ace953f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.node.usage; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java index 261b9eaec8927..59b1f99429cfe 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java @@ -36,8 +36,8 @@ import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java index dbd3673149efe..7a4aeff50c6e7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.aggregations.support.AggregationUsageService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoRequest.java index e022ef504bd46..be219b15875df 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoRequest.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoResponse.java index bc025ec64bbdc..8173da8455434 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remote/RemoteInfoResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.transport.RemoteConnectionInfo; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java index 6d05e8cbee986..703b9575a88ad 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java @@ -11,8 +11,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java index e3ede1d51f10e..4ff03cb880cf3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java @@ -11,13 +11,13 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.RestoreInfo; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/TransportRestoreRemoteStoreAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/TransportRestoreRemoteStoreAction.java index 7304ba25717ac..613bf078b2c96 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/TransportRestoreRemoteStoreAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/TransportRestoreRemoteStoreAction.java @@ -18,7 +18,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.snapshots.RestoreService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index 117ad8ac2bf59..5ac9c1cf5f74c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -8,9 +8,9 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java index 5de14f77a7c37..f09cf79c5154c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java @@ -9,8 +9,8 @@ package org.opensearch.action.admin.cluster.remotestore.stats; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java index 5a49f90b42b07..20023e30a271e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java @@ -8,11 +8,11 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java index ff974e32678d6..434abd1207f50 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java @@ -9,7 +9,7 @@ package org.opensearch.action.admin.cluster.remotestore.stats; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -21,7 +21,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java index 0f265681cd241..7e3f4cd95fc72 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java index 0b5ebc967bb3e..7d9dcd25aca8c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index ec02173ff2807..2b91c6641211d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -51,7 +51,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java index a3f4bb768c649..352a3772e039b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index 08e3bc6df0d83..05a95b926f576 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.repositories.RepositoriesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java index ca70dcce9d0c0..fb1f79dab4e39 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java index ca9b2a702df31..53a94a8a56773 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -45,7 +45,7 @@ import java.util.Collections; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Get repositories response diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java index de942ef284f3b..d367f75607d36 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index d57fd04b30eaa..b001e9456e78c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java index 6a5be14df93fd..16f7d6d5700bf 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.repositories.RepositoriesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java index 5215078f52d3b..ec54d7640c1d3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.repositories.RepositoriesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java index 001030f6a67f5..b84161e716f5d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java index d1c19b5a40cff..8b9142a39e9c9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java index 806fa80691202..a6addce14787d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.cluster.routing.allocation.command.AllocationCommand; import org.opensearch.cluster.routing.allocation.command.AllocationCommands; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index d894026124273..a62029218ca25 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.allocation.RoutingExplanations; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index 94c4269e77109..0a4b98f9d4bb5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -60,7 +60,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java index c8eca7acb6017..8d29baa82562a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index c272a19dc6985..aaa89ee269fd8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -35,8 +35,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java index 463a77a0dcf9a..6daa2a76482dd 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index e9cb6a78f5269..542ca612d2b6c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -53,7 +53,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java index 039b1deb6ca7c..0ee7e8d67d307 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java @@ -33,12 +33,12 @@ package org.opensearch.action.admin.cluster.shards; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java index fd758fe8f4d91..e1d88faa10082 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java index 21c1b29adc324..d43ef8a0c9c61 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.internal.AliasFilter; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 2f7c194e0acd7..7e20e6a1b31d5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.search.internal.AliasFilter; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java index 738c8a9d514c9..ad6b035d1f51e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java @@ -15,9 +15,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java index b98ac6c0c55be..2a417e9f4287f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java @@ -9,8 +9,8 @@ package org.opensearch.action.admin.cluster.shards.routing.weighted.delete; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/TransportDeleteWeightedRoutingAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/TransportDeleteWeightedRoutingAction.java index 8f88d8af71b70..903fbb13d9f45 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/TransportDeleteWeightedRoutingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/TransportDeleteWeightedRoutingAction.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.routing.WeightedRoutingService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java index aaa000baa95f3..7dcec15c750fc 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java index edbfa95b228a0..91bd9ac2f00d1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java @@ -13,8 +13,8 @@ import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.WeightedRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -24,7 +24,7 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Response from fetching weights for weighted round-robin search routing policy. diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/TransportGetWeightedRoutingAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/TransportGetWeightedRoutingAction.java index 280fca29944e3..e73e662314378 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/TransportGetWeightedRoutingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/TransportGetWeightedRoutingAction.java @@ -24,7 +24,7 @@ import org.opensearch.cluster.routing.WeightedRoutingService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java index fe95b614a02d1..bf1f42aedc4bb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java @@ -16,9 +16,9 @@ import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.WeightedRouting; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java index b0154aceef0c2..cbf10aa74f8a2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java @@ -9,7 +9,7 @@ package org.opensearch.action.admin.cluster.shards.routing.weighted.put; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/TransportAddWeightedRoutingAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/TransportAddWeightedRoutingAction.java index 56203d4edc3d0..ed178884faf54 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/TransportAddWeightedRoutingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/TransportAddWeightedRoutingAction.java @@ -19,7 +19,7 @@ import org.opensearch.cluster.routing.WeightedRoutingService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java index 6182bf153a519..fedcfa1f5d9ff 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java index e9f5153f78700..05b3a5d676c5b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.snapshots.SnapshotsService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 0226b20b5bf34..5da3f2eb01260 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -39,9 +39,9 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java index d88dd1578231b..bef95c417c038 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java @@ -35,13 +35,13 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotInfo.SnapshotInfoBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index f604a30121797..79f365bd3a951 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.snapshots.SnapshotsService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java index 832b37050ffe6..d08c3033e7e10 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index c78968c2a0848..04b2c0b1be1b1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.snapshots.SnapshotsService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index 5aa695f9bc1dd..7492b5a434fe0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index a015010065d0f..3c96c67f78fd4 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index b5445bf544cc6..11ea4112f6e67 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java index 64538226ccf9e..6fd593fdd3e08 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.ClusterStateListener; import org.opensearch.cluster.RestoreInProgress; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.RestoreInfo; import org.opensearch.snapshots.RestoreService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index cddb338310c4f..840564a4bd7a2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java index cbf478f4c5485..2cf1b40b92761 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java @@ -35,14 +35,14 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.RestoreInfo; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index c2f79b2a27157..ec4d6b8412318 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.snapshots.RestoreService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index 62231f0edfed6..070f77f603621 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -36,16 +36,16 @@ import org.opensearch.action.support.broadcast.BroadcastShardResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java index 3d7a89c63da14..9c2db62c33bd0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java @@ -38,7 +38,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java index eb16b9f51fd3f..282585a43183a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -33,16 +33,16 @@ package org.opensearch.action.admin.cluster.snapshots.status; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java index c830c30018efc..a3b401980b109 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.SnapshotsInProgress.State; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index 5573f36e12a54..6aeff1d980f0c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java index c96c71eb8e566..bb7bb7eb70b68 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java index e9bf564afaf32..8f76dbce048cd 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java @@ -43,9 +43,9 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotShardsService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index fb1c97fc884a0..78952805e5b49 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -47,11 +47,11 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java index 5454cfb72e9c1..bc6724d3081d6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java index f65d15c5c64aa..ce3b020280b16 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java index 15c2cea31f530..e57d2cf1b9803 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.node.NodeClosedException; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java index 2ce7164021a3a..9cdd5bf244ecb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index 2d684c3333f24..1b25bf84356d6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 43b3cf11237f7..dc472c10f550b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.stats; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java index 0306ca0c94cfe..1dda39a17babc 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java index 4a786aa831e16..f833c52493e00 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.cluster.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java index 4d5adf9092db5..90c78f30ea78d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index aee6dfddd203e..18098bc31432f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -46,8 +46,8 @@ import org.opensearch.cluster.health.ClusterStateHealth; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.IndexService; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.seqno.RetentionLeaseStats; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java index 93d2c3ba3c452..7d92162015950 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java index 385948b613199..07d8530eb7ef6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java index be90358aade20..f269a813dbaa4 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java @@ -34,13 +34,13 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.ScriptContextInfo; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java index b6159e10cecb7..a81ca39595dfb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java index ba09cadce913c..192e89e08f25d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java @@ -33,13 +33,13 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.ScriptLanguagesInfo; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java index bdff2e27b0f2d..70384b5fb648e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java index 5bd189f8b69b2..02bee957e6216 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java @@ -34,14 +34,14 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.StoredScriptSource; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 583f208287e68..ecfcd6310dd0a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -34,9 +34,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java index ef3c14df29627..ed46b12d96106 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java @@ -35,7 +35,7 @@ import org.opensearch.action.support.master.AcknowledgedRequestBuilder; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; /** diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java index e41ec2b1f737c..f126a8474a456 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java index 8dbadf34ab06b..7b8a2805bf1a6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java index 8ffe4d2b74695..ac518291bd930 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java index ec9a830abf1b8..83e8b93b32e0f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java index aa725a8aacc00..f0d75ce77cd8a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.service.PendingClusterTask; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java index abe93fc45125a..0bd761ca811f3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.cluster.service.PendingClusterTask; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java index 2d2f36d3d0eaa..9118cdd56babd 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java @@ -36,10 +36,10 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java index 234b375452dfb..142cbe6a0ab0b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -39,11 +39,11 @@ import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.cluster.metadata.AliasAction; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index e8f4f9764dd47..bc8cafce98ff3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -51,8 +51,8 @@ import org.opensearch.cluster.metadata.MetadataIndexAliasesService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java index c7a7e16ef112c..5833f14c6e0d3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -35,8 +35,8 @@ import org.opensearch.action.AliasesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java index 02e999c797e3d..7408500f4f2a0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.metadata.AliasMetadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index a30074588cc42..92301e962a55f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.indices.SystemIndices; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java index 014d28bbfce09..37e05151c8179 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java @@ -37,9 +37,9 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 2700d38686833..97e2bae33f3e3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -49,7 +49,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexService; @@ -64,7 +64,7 @@ import org.opensearch.index.analysis.TokenizerFactory; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.StringFieldType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java index 56f1e59749f66..786c5a9c86ca7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java @@ -34,8 +34,8 @@ import org.opensearch.Version; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java index 021c1805a6fac..449993246d808 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.indices.cache.clear; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index 824f5dbdc5349..a371047873d11 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.cache.clear; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.IndicesService; import org.opensearch.node.Node; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java index 1095cec447442..8c6dde80d8d97 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java index b806e78cd93ff..8e58ee92db80a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java @@ -32,18 +32,18 @@ package org.opensearch.action.admin.indices.close; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java index c4c789a8de90e..b4c9ad30e4e84 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -46,12 +46,12 @@ import org.opensearch.cluster.metadata.MetadataIndexStateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java index 8e2691f2d8e19..b1f3fb913b250 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java @@ -45,12 +45,12 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.TaskId; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java index 556d46683f0ab..22e0c8aedf4f5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java @@ -55,7 +55,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index aba5a60ef2804..ad45e5346f9fa 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.block.ClusterBlock; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.HashSet; import java.util.Set; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java index b9c11a003c459..53840b7697e45 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java @@ -43,10 +43,10 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 163e0c8450c1c..a716959614065 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.master.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java index 06fe9d5528b4e..1b3ad48402eed 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java index bccff6849de4d..c8a51da38662f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.MetadataCreateIndexService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/DanglingIndexInfo.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/DanglingIndexInfo.java index cd0feae4428db..47ba1cca9fc4a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/DanglingIndexInfo.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/DanglingIndexInfo.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.indices.dangling; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java index 4fad5498de375..0443325f82778 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java index 23f5f3e177df6..d5801223855d2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java @@ -57,9 +57,9 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java index f9c6d8d2f1d50..ae284d79bb085 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.dangling.find; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java index 8c99ad8c55f10..b834775be1767 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java index 6885de74e4479..8b0d6df497cc8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java @@ -34,8 +34,8 @@ import java.io.IOException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; /** diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java index cfe71a7acf9a3..41504a436750a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java index 3119625c6b796..84d044a303a2c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.gateway.DanglingIndicesState; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java index 73fbad248b8b1..590f08a82c1d2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java index a361d321bfd46..a28320f5dbbee 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.dangling.list; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java index 1037b0d8dead2..f785085da9a03 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java @@ -36,12 +36,12 @@ import org.opensearch.action.admin.indices.dangling.DanglingIndexInfo; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java index 696daf75942fb..d8367239815f3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.dangling.list; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java index 5f0fa78ff231a..338f2f79235b8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.admin.indices.dangling.DanglingIndexInfo; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java index 6641ff172c2f0..59076cc6ef8fb 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.gateway.DanglingIndicesState; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java index 9f1069e60c625..2a53eb38d672c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -49,8 +49,8 @@ import org.opensearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java index 02c0966a8f605..0b2375850f1fc 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java @@ -37,7 +37,7 @@ import org.apache.lucene.index.PointValues; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastRequest; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; @@ -52,9 +52,9 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java index ca032e66a34f7..e5ae7cd582481 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java @@ -55,13 +55,13 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotsService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java index 236effbf69a85..543783b3de367 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java @@ -55,12 +55,12 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java index 7475121a910c4..35fb6a8748941 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java index bcf2f6f534d74..a91ca1a7b714c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -47,8 +47,8 @@ import org.opensearch.cluster.metadata.MetadataDeleteIndexService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java index 294f395437931..6d2bea438f3ff 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java index 5bf51fe2ba653..db1a8620be5aa 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.exists.indices; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java index f5f7e0e9ea7b7..1469b48d04fc7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexNotFoundException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java index 0ff502dee8396..c8b28efc5f294 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java index 4e115a782af20..85ca20c30c08b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.indices.flush; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/ShardFlushRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/ShardFlushRequest.java index f56a5cc6dd5e2..ab6e99064b43c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/flush/ShardFlushRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/ShardFlushRequest.java @@ -34,9 +34,9 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.replication.ReplicationRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportFlushAction.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportFlushAction.java index 28faed924de6e..5a114a6765dbc 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportFlushAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportFlushAction.java @@ -33,13 +33,13 @@ package org.opensearch.action.admin.indices.flush; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.action.support.replication.TransportBroadcastReplicationAction; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.TransportService; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportShardFlushAction.java index 5a5a57b4bcc00..c96a55a32aee7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -40,11 +40,11 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java index 20b636dfd1e21..89e5a57094a96 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java @@ -35,8 +35,8 @@ import org.opensearch.Version; import org.opensearch.action.support.broadcast.BroadcastRequest; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.engine.Engine; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java index c57c7cf58f95f..4b5825724df84 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.indices.forcemerge; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/TransportForceMergeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/TransportForceMergeAction.java index dc48e3c93176a..e3afe420d0a18 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/TransportForceMergeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/TransportForceMergeAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.forcemerge; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java index 9a7fae9f84a98..f5a2a213bd720 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.info.ClusterInfoRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.ArrayUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java index a44fcb93966bb..df05c3dd665d2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java index 558f4f513966b..66d810b091c4c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java index 1ce2c2473c455..08b4f622425bc 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.single.shard.SingleShardRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java index 55a36a95dfd2d..811ace0082dfb 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index a9e5e8f1484b8..a06121b1d448d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -35,9 +35,9 @@ import org.opensearch.Version; import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java index 2c9bec8398b66..3988b0dd5a508 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.info.ClusterInfoRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java index 757e6e350a60b..c4c9094e276d6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 4def2071d84a1..435034c77b921 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -42,9 +42,9 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.regex.Regex; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentHelper; @@ -53,7 +53,7 @@ import org.opensearch.index.mapper.MappingLookup; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java index 933d598cd4dd8..61b6f67ebdabb 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java index 200c12fd90a72..19e6664956354 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -40,10 +40,10 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; @@ -52,7 +52,7 @@ import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.MapperService; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java index 7a16974b70fb5..a1300b5859ce5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java @@ -38,7 +38,7 @@ import org.opensearch.client.OpenSearchClient; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index c4dad614c53dd..0b66689b869f1 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -42,8 +42,8 @@ import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index de546f428bafa..6c3482da3cac0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -48,8 +48,8 @@ import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java index 9bc6395c0ea94..d42f3699765e7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java index 7d5ac66edaafe..bd96a1071c129 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.open; import org.opensearch.action.support.master.ShardsAcknowledgedResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java index 2ccb5f6d22886..018c527f3d759 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -47,8 +47,8 @@ import org.opensearch.cluster.metadata.MetadataIndexStateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java index 7d208b5e0ac77..ca5a17f0a1520 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java index 6d89170fefd7c..39268d752c8c5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java @@ -32,18 +32,18 @@ package org.opensearch.action.admin.indices.readonly; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java index 560d2e6389c63..ca1ca2c5f33b9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java @@ -46,8 +46,8 @@ import org.opensearch.cluster.metadata.MetadataIndexStateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java index b295977933b25..3dc459a380bfd 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java @@ -45,12 +45,12 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.TaskId; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java index d89c86a8f8e79..5294e573bab7c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java index ae28cf591b12b..7664a73c27fc8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java @@ -32,11 +32,11 @@ package org.opensearch.action.admin.indices.recovery; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.indices.recovery.RecoveryState; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/opensearch/action/admin/indices/recovery/TransportRecoveryAction.java index 132354ed83b6c..1f76ee4bc5337 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.recovery; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java index 4f078f8d9a23d..c6e230cc66373 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.refresh; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java index 5f913d6e08e4d..8c83986e384c1 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.indices.refresh; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportRefreshAction.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportRefreshAction.java index 7fb4a9908c422..ae56160b098a4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -34,14 +34,14 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.replication.BasicReplicationRequest; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.action.support.replication.TransportBroadcastReplicationAction; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.TransportService; import java.util.List; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshAction.java index 9c955c8157459..1541753581c95 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java index b6855507669b6..ce17176a220ae 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java @@ -9,9 +9,9 @@ package org.opensearch.action.admin.indices.replication; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.indices.replication.SegmentReplicationState; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java index 6df990aecd3e3..9d8caf1d7b20b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java index dcff7ddc1d7ef..63899668badca 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java @@ -8,11 +8,11 @@ package org.opensearch.action.admin.indices.replication; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.SegmentReplicationPerGroupStats; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index f8d5156ff1192..8d299be0bea8b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -9,7 +9,7 @@ package org.opensearch.action.admin.indices.replication; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -19,13 +19,13 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java index 413c97d641de1..eb85268961571 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -52,9 +52,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.common.Strings; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java index 3b490957ee3ec..280dc307447b7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.rollover; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxAgeCondition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxAgeCondition.java index dda51536ada3a..3752ee6ece82c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxAgeCondition.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxAgeCondition.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.rollover; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxDocsCondition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxDocsCondition.java index 763ed07cf9f16..9c7684e25e543 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxDocsCondition.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxDocsCondition.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.rollover; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java index d4416850691af..f0ab571ea9f75 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.rollover; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java index 5a23b680637ed..f58c842be374b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java @@ -35,9 +35,9 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java index ef4b281ab24a1..95a4b6573611d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java @@ -38,8 +38,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java index 371af7220d795..55ee65d0a4973 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java index b4ecae2ee08ba..4ddff1563885a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -54,7 +54,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.index.shard.DocsStats; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java index e0633d3d557ae..a6caf0649fde1 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.segments; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Arrays; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java index 784fab666127a..5604b13e7a2a4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -36,10 +36,10 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.engine.Segment; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java index 948b95b5a77be..412568cdefc10 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.segments; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java index ef1704dedf414..e66717cadb464 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java index 7214d0203966c..90317542244ff 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java @@ -33,9 +33,9 @@ package org.opensearch.action.admin.indices.segments; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.engine.Segment; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java index 163bd745cf029..b6dc3ddb19081 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.segments; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java index d843b3a9452a5..276551372339f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java @@ -13,7 +13,7 @@ import org.opensearch.action.search.SearchContextId; import org.opensearch.action.search.SearchContextIdForNode; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -28,12 +28,12 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.search.SearchService; import org.opensearch.search.internal.PitReaderContext; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java index 37587bd5e4c25..9666bcb34831a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java index 86008630077ec..05b06fc2b62c7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java @@ -34,14 +34,14 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.json.JsonXContent; import java.io.ByteArrayOutputStream; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java index 671d04fde5bcb..22cbcf804f9d2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java @@ -42,13 +42,13 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.common.util.CollectionUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 66be780d22e03..465c1dd573567 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -48,8 +48,8 @@ import org.opensearch.cluster.metadata.MetadataUpdateSettingsService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index 335fe20b518c1..a7b7e005bce90 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java index a910e849fe87a..ea3de86fa17c0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.cluster.health.ClusterHealthStatus; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java index 01321bb09323d..37d1ab07605b2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java @@ -35,12 +35,12 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.collect.ImmutableOpenIntMap; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index e0fb4fd922ef6..cbb5c3d686cba 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -56,12 +56,12 @@ import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.gateway.AsyncShardFetch; import org.opensearch.gateway.TransportNodesListGatewayStartedShards; import org.opensearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java index 78636fd04984b..b2e7ed92e608a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java @@ -40,8 +40,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java index 69e973d39dc09..1aa09023e3583 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.shrink; import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java index fe60a82d0e76f..0d31c90a98f56 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java @@ -49,12 +49,12 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.DocsStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import org.opensearch.common.unit.ByteSizeValue; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java index f88508291ef9e..5a3a34e9a2ebe 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java @@ -34,9 +34,9 @@ import org.apache.lucene.store.AlreadyClosedException; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java index c93299e350ca6..502b90417615f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java @@ -33,9 +33,9 @@ package org.opensearch.action.admin.indices.stats; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java index 6a423cf8d15ea..1635ce0bf83fc 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java @@ -32,10 +32,10 @@ package org.opensearch.action.admin.indices.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java index 06315377797d9..54f3e9b7d1a24 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.stats; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java index 6efe870185497..4014bad06ff9a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -33,15 +33,15 @@ package org.opensearch.action.admin.indices.stats; import org.opensearch.action.admin.indices.stats.IndexStats.IndexStatsBuilder; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java index 80806d668cd0b..77562fa19b319 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java @@ -34,9 +34,9 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.engine.CommitStats; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 429c6630c6c06..9bb519c175f9d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -34,7 +34,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.seqno.RetentionLeaseStats; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java index 38c0ce1b7faf8..e719ad9bdd174 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java index a91f89f55420e..7bfec26e924dd 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java index cf8c2762990d1..85524bddc56d8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java index 75cc8ffe05f73..326f1c8ec059c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.metadata.MetadataIndexTemplateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index 52464dbd90e3f..3f071c4074074 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.metadata.MetadataIndexTemplateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index 3d3c2bd0613f2..8503d7fea6e51 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.metadata.MetadataIndexTemplateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateAction.java index a386fdd186fd6..ff038bd89a597 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateAction.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.metadata.ComponentTemplate; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java index b30041fc455f6..2902bd5b81eba 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java index e97c97d78b6e6..1ac0f7e2d45a7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 8cd9d5cc36ed1..d3be5de84d758 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.metadata.IndexTemplateMetadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java index 36e3a1d0e6264..1ad2dc1636d40 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java index 327a40be64a2a..0bf5d64905a98 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java index 2a511e60eaf7b..c99e7aac09b98 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java index 2dceb0bb866cd..66fcf0e7cf0ae 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 0800ed4a1362a..76c5a4cbd669c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.metadata.Template; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateAction.java index 210e8030093e1..0055676a7051a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateAction.java @@ -38,8 +38,8 @@ import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d66cee0802f78..70af69bc77760 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -52,7 +52,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.mapper.DocumentMapper; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 4238b8fc7027f..d50a8b199230a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java index 594bde675239a..62fe778d3f922 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -38,8 +38,8 @@ import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.metadata.ComponentTemplate; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java index bd2ca61cc94c7..63c3b492ff705 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java @@ -41,8 +41,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.regex.Regex; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 0ab896bf87750..099f7c34ff818 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -42,10 +42,10 @@ import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index ac028c2f17ebb..eae2f9d9c94e0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -35,7 +35,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 925913c4e8d3e..e67062123ae16 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.metadata.Template; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index 20ba5376f1add..2cc8dfbefed82 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.metadata.MetadataIndexTemplateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index c93d83cd5e70d..826aa888dbacf 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.metadata.MetadataIndexTemplateService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java index df937fab349ec..32cf6d4bfe70d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.upgrade.get; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Arrays; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java index 8ca8188899d02..57fb2513faf78 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java @@ -34,8 +34,8 @@ import org.opensearch.action.support.broadcast.BroadcastShardResponse; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java index c81d86660ae50..1d21ed8569e2f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.engine.Segment; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java index f3f0bc70ce201..258010d29e828 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.upgrade.get; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java index ce2333b2afb56..7fe663a347ee3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java @@ -32,10 +32,10 @@ package org.opensearch.action.admin.indices.upgrade.get; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/ShardUpgradeResult.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/ShardUpgradeResult.java index 1f251f2d8cda4..ebcb269725a6d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/ShardUpgradeResult.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/ShardUpgradeResult.java @@ -33,10 +33,10 @@ package org.opensearch.action.admin.indices.upgrade.post; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.text.ParseException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java index e50c8a94d5e20..5b6bd7414b154 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.PrimaryMissingActionException; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; @@ -51,7 +51,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java index df0d5cf57e7de..a73110c781ba7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java @@ -47,7 +47,7 @@ import org.opensearch.cluster.metadata.MetadataUpdateSettingsService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java index fc75da23c805e..98c307c37ea54 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.upgrade.post; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java index b763c893f9bdf..f6d2bac786b5f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java @@ -33,11 +33,11 @@ package org.opensearch.action.admin.indices.upgrade.post; import org.opensearch.Version; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java index 618c2114babc3..306d29dd84f13 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java index debdf925c504e..248cab5e40eaf 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java @@ -33,9 +33,9 @@ package org.opensearch.action.admin.indices.validate.query; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java index 7df5a27929778..9e1e3c6b1e1e7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java @@ -34,10 +34,10 @@ import org.opensearch.Version; import org.opensearch.action.support.broadcast.BroadcastShardRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.internal.AliasFilter; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java index 7da73f5038e92..f72352cd8cd95 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java @@ -33,9 +33,9 @@ package org.opensearch.action.admin.indices.validate.query; import org.opensearch.action.support.broadcast.BroadcastShardResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 431c5468d1850..7919b16aa3af3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -36,7 +36,7 @@ import org.apache.lucene.search.Query; import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.opensearch.action.support.broadcast.TransportBroadcastAction; import org.opensearch.cluster.ClusterState; @@ -46,10 +46,10 @@ import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Randomness; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lease.Releasables; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.ParsedQuery; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java index 91485d84eceb3..a3fa560b78d65 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java index 965ecda4a4bfd..1d73d4821f2b4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java @@ -32,11 +32,11 @@ package org.opensearch.action.admin.indices.validate.query; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkItemRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkItemRequest.java index 4a40d277b5c17..2a85b7abb741a 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkItemRequest.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkItemRequest.java @@ -37,11 +37,11 @@ import org.opensearch.action.DocWriteRequest; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java index c712a18e85013..26cd318e7a280 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java @@ -42,9 +42,9 @@ import org.opensearch.action.update.UpdateResponse; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; @@ -54,13 +54,13 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.throwUnknownField; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.throwUnknownField; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java index 0ee289e8f5af7..4695b44c4986b 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java @@ -38,7 +38,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Client; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java index 2c42b0b8a74b0..07d29413667cc 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java @@ -45,10 +45,10 @@ import org.opensearch.action.support.replication.ReplicationRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java index 3f3d1d550afff..569aa8f9e4f34 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java @@ -38,8 +38,8 @@ import org.opensearch.action.update.UpdateRequest; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java b/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java index b8e1a03a2bc29..f013749e380f0 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java @@ -33,13 +33,13 @@ package org.opensearch.action.bulk; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; @@ -47,9 +47,9 @@ import java.util.Iterator; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.throwUnknownField; -import static org.opensearch.common.xcontent.XContentParserUtils.throwUnknownToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.throwUnknownField; +import static org.opensearch.core.xcontent.XContentParserUtils.throwUnknownToken; /** * A response of a bulk execution. Holding a response for each item responding (in order) of the diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java index 484f57abb8b07..c86f8c87eee15 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java @@ -36,9 +36,9 @@ import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.action.support.replication.ReplicatedWriteRequest; import org.opensearch.action.support.replication.ReplicationRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.HashSet; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkShardResponse.java b/server/src/main/java/org/opensearch/action/bulk/BulkShardResponse.java index cfedcde92194c..f1503eaff2da6 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkShardResponse.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkShardResponse.java @@ -35,9 +35,9 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.support.WriteResponse; import org.opensearch.action.support.replication.ReplicationResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/bulk/MappingUpdatePerformer.java b/server/src/main/java/org/opensearch/action/bulk/MappingUpdatePerformer.java index 688912afe0537..0f2e4f1b00763 100644 --- a/server/src/main/java/org/opensearch/action/bulk/MappingUpdatePerformer.java +++ b/server/src/main/java/org/opensearch/action/bulk/MappingUpdatePerformer.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.index.mapper.Mapping; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /** * Updates the mappings on the cluster manager diff --git a/server/src/main/java/org/opensearch/action/bulk/Retry.java b/server/src/main/java/org/opensearch/action/bulk/Retry.java index fdd4447144008..754ee9ba4dddc 100644 --- a/server/src/main/java/org/opensearch/action/bulk/Retry.java +++ b/server/src/main/java/org/opensearch/action/bulk/Retry.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.common.unit.TimeValue; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.RemoteTransportException; diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java index ed407550b8b2f..f6ca9022a5bff 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java @@ -72,12 +72,12 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.VersionType; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.SystemIndices; import org.opensearch.ingest.IngestService; diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java index cbb30714ee8e1..0b675fd9125fa 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java @@ -67,12 +67,12 @@ import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; @@ -92,7 +92,7 @@ import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java index e5f96fdab9c96..707cf62d87beb 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java @@ -41,7 +41,7 @@ import org.opensearch.action.support.WriteResponse; import org.opensearch.action.support.replication.ReplicatedWriteRequest; import org.opensearch.action.support.replication.ReplicationResponse; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java b/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java index b9bd556688ca1..aebff808286ec 100644 --- a/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java @@ -39,13 +39,13 @@ import org.opensearch.action.DocWriteRequest; import org.opensearch.action.support.replication.ReplicatedWriteRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.core.common.Strings; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java b/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java index 35823cc348cdc..35a111e456db5 100644 --- a/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java +++ b/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java @@ -33,14 +33,14 @@ package org.opensearch.action.delete; import org.opensearch.action.DocWriteResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * The response of the delete action. diff --git a/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java b/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java index 8e20a921c7b3f..c6792449042dc 100644 --- a/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java +++ b/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java b/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java index 783bc78c3e5d5..6d0fbfd15bee6 100644 --- a/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java +++ b/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java @@ -36,15 +36,15 @@ import org.opensearch.Version; import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.get.GetResult; import org.opensearch.index.mapper.MapperService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/opensearch/action/explain/TransportExplainAction.java index 5752860af4f16..4fd3ab8dcf389 100644 --- a/server/src/main/java/org/opensearch/action/explain/TransportExplainAction.java +++ b/server/src/main/java/org/opensearch/action/explain/TransportExplainAction.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lease.Releasables; import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; @@ -52,7 +52,7 @@ import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchService; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.SearchContext; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java index e898bb38193d6..8053e1f8521cf 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java @@ -33,9 +33,9 @@ package org.opensearch.action.fieldcaps; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java index 2b6a89c66ef4b..c4dead2a3de25 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java @@ -37,10 +37,10 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.OriginalIndices; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java index 4da5d04fe9d7a..f3ca0c4b192ed 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java @@ -33,9 +33,9 @@ package org.opensearch.action.fieldcaps; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java index 4065fc8f4be33..add0377d9be48 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.ValidateActions; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java index 0429e77cbae89..550ef2d412ca4 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -36,13 +36,13 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/opensearch/action/fieldcaps/IndexFieldCapabilities.java index 73025c2eac3f0..8a5cd13460c00 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/IndexFieldCapabilities.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/IndexFieldCapabilities.java @@ -32,9 +32,9 @@ package org.opensearch.action.fieldcaps; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 7fa9e43bee21d..acabc12af0e56 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -55,13 +55,13 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.logging.LoggerMessageFormat; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.index.query.MatchAllQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.search.SearchService; import org.opensearch.search.builder.SearchSourceBuilder; diff --git a/server/src/main/java/org/opensearch/action/get/GetRequest.java b/server/src/main/java/org/opensearch/action/get/GetRequest.java index ff0450404f7f9..81102609f4c2d 100644 --- a/server/src/main/java/org/opensearch/action/get/GetRequest.java +++ b/server/src/main/java/org/opensearch/action/get/GetRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.RealtimeRequest; import org.opensearch.action.ValidateActions; import org.opensearch.action.support.single.shard.SingleShardRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.core.common.Strings; import org.opensearch.index.VersionType; diff --git a/server/src/main/java/org/opensearch/action/get/GetResponse.java b/server/src/main/java/org/opensearch/action/get/GetResponse.java index 56816b3b59c9f..b713dc8a507d1 100644 --- a/server/src/main/java/org/opensearch/action/get/GetResponse.java +++ b/server/src/main/java/org/opensearch/action/get/GetResponse.java @@ -34,12 +34,12 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.action.ActionResponse; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java b/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java index dcc296c4b6e0b..19c9b785e7ea2 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.get; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java b/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java index dee4db4f18a0c..e8d0c1b9d320f 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java @@ -42,11 +42,11 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java b/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java index 88f78d2ca6815..563345caaf1f8 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java @@ -35,9 +35,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java b/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java index 23ebf91fe2300..c32b1da068e67 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetShardRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.single.shard.SingleShardRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java b/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java index 93adaa66f437d..2425282c727a6 100644 --- a/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java +++ b/server/src/main/java/org/opensearch/action/get/MultiGetShardResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.get; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/get/TransportGetAction.java b/server/src/main/java/org/opensearch/action/get/TransportGetAction.java index 6d6798fb64178..07546034665b3 100644 --- a/server/src/main/java/org/opensearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/opensearch/action/get/TransportGetAction.java @@ -41,11 +41,11 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.IndexService; import org.opensearch.index.get.GetResult; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/get/TransportMultiGetAction.java b/server/src/main/java/org/opensearch/action/get/TransportMultiGetAction.java index 1e0566e23fff4..6fe73fddc27a6 100644 --- a/server/src/main/java/org/opensearch/action/get/TransportMultiGetAction.java +++ b/server/src/main/java/org/opensearch/action/get/TransportMultiGetAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.util.concurrent.AtomicArray; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/opensearch/action/get/TransportShardMultiGetAction.java index e882ffdc8c344..b0596ac2fb448 100644 --- a/server/src/main/java/org/opensearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/opensearch/action/get/TransportShardMultiGetAction.java @@ -42,11 +42,11 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.IndexService; import org.opensearch.index.get.GetResult; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequest.java b/server/src/main/java/org/opensearch/action/index/IndexRequest.java index 1cce14ef447f5..86ba13dedfa06 100644 --- a/server/src/main/java/org/opensearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/opensearch/action/index/IndexRequest.java @@ -48,10 +48,10 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentFactory; @@ -62,7 +62,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java index 4f9536be6afcb..ca25409556398 100644 --- a/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java @@ -37,7 +37,7 @@ import org.opensearch.action.support.replication.ReplicationRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; diff --git a/server/src/main/java/org/opensearch/action/index/IndexResponse.java b/server/src/main/java/org/opensearch/action/index/IndexResponse.java index 75de7d7b33887..12d788323b497 100644 --- a/server/src/main/java/org/opensearch/action/index/IndexResponse.java +++ b/server/src/main/java/org/opensearch/action/index/IndexResponse.java @@ -34,15 +34,15 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A response of an index operation, diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java index 0bd102849eee8..2f05ce3a25320 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java index 9085b2347765c..86c4c09303c15 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.ingest.IngestService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java index f6bbf44a09782..bbe56aa0ff9dd 100644 --- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java index 6890252cec014..a26fa413b2f5b 100644 --- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java +++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java @@ -34,16 +34,16 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.ingest.PipelineConfiguration; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; @@ -52,7 +52,7 @@ import java.util.List; import java.util.Map; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * transport response for getting a pipeline diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java index 5a59c8255361e..1283a68a3ea5f 100644 --- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.ingest.IngestService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java index e2cbba531cee3..a3067fe0ba160 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java @@ -34,9 +34,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java index fec2cdef089e4..d69165b280063 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionRequestBuilder; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; /** diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java index 61a2deedfd511..f87d832078661 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.ingest.IngestInfo; import org.opensearch.ingest.IngestService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentBaseResult.java b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentBaseResult.java index db88214cff0ec..13cec3c8c46a6 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentBaseResult.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentBaseResult.java @@ -33,8 +33,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java index 3e698374cd2e2..98a03272aff42 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java @@ -31,7 +31,7 @@ package org.opensearch.action.ingest; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; /** diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentVerboseResult.java b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentVerboseResult.java index 0987325f2501a..f26bb106de273 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentVerboseResult.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentVerboseResult.java @@ -32,8 +32,8 @@ package org.opensearch.action.ingest; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java index a049a09abdd22..31f71e5720daa 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java @@ -34,9 +34,9 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java index 0cf71b66fe936..b2eda0e9485e4 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestBuilder; import org.opensearch.client.OpenSearchClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; /** diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java index e1b841a8b530f..9e67f791dc141 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java @@ -35,8 +35,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -49,7 +49,7 @@ import java.util.List; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * transport response for simulating a pipeline diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java index daffbbf61a1bb..c1dc1be97455e 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.ingest.IngestService; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/opensearch/action/ingest/SimulateProcessorResult.java index 509705fe856e8..7b04f96a220bd 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulateProcessorResult.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/opensearch/action/ingest/WriteableIngestDocument.java index 7e38cd89c0f50..f8cc4736e87b0 100644 --- a/server/src/main/java/org/opensearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/opensearch/action/ingest/WriteableIngestDocument.java @@ -33,9 +33,9 @@ package org.opensearch.action.ingest; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/main/MainRequest.java b/server/src/main/java/org/opensearch/action/main/MainRequest.java index e1c1844f9d4dc..459633fa63cb3 100644 --- a/server/src/main/java/org/opensearch/action/main/MainRequest.java +++ b/server/src/main/java/org/opensearch/action/main/MainRequest.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/main/MainResponse.java b/server/src/main/java/org/opensearch/action/main/MainResponse.java index e51ef093817f4..30994c5cfcbd2 100644 --- a/server/src/main/java/org/opensearch/action/main/MainResponse.java +++ b/server/src/main/java/org/opensearch/action/main/MainResponse.java @@ -37,8 +37,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java b/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java index 0cbca15232e44..6a4f2f0607144 100644 --- a/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java @@ -32,9 +32,9 @@ package org.opensearch.action.resync; import org.opensearch.action.support.replication.ReplicatedWriteRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/resync/ResyncReplicationResponse.java b/server/src/main/java/org/opensearch/action/resync/ResyncReplicationResponse.java index 22507ee717a1c..d1c7be3673645 100644 --- a/server/src/main/java/org/opensearch/action/resync/ResyncReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/resync/ResyncReplicationResponse.java @@ -33,7 +33,7 @@ import org.opensearch.action.support.WriteResponse; import org.opensearch.action.support.replication.ReplicationResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java index 2249c7cace943..879250218405c 100644 --- a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.engine.Engine; diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index 48fac9e8c8d38..c67b56fed159e 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -39,7 +39,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.action.NoShardAvailableActionException; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.action.support.TransportActions; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.FailAwareWeightedRouting; @@ -50,7 +50,7 @@ import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.AliasFilter; diff --git a/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java b/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java index a5830ba63730a..cd9e33634d918 100644 --- a/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java +++ b/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java b/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java index 55e7f47748176..b6cd5d5bb7c0e 100644 --- a/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java +++ b/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java @@ -34,20 +34,20 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; /** * Transport response for clearing a search scroll diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitController.java b/server/src/main/java/org/opensearch/action/search/CreatePitController.java index 2130c1f91b59d..8d2ea3a03266c 100644 --- a/server/src/main/java/org/opensearch/action/search/CreatePitController.java +++ b/server/src/main/java/org/opensearch/action/search/CreatePitController.java @@ -19,11 +19,11 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java b/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java index a679a66eb4acc..f496a22caff6d 100644 --- a/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java +++ b/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java @@ -13,8 +13,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java b/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java index dd17d7debdeaf..2fb10cfeb727c 100644 --- a/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java +++ b/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java @@ -10,19 +10,19 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActions; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Create point in time response with point in time id and shard success / failures diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java b/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java index 6a08348e4678e..1e616ab5ca16e 100644 --- a/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java +++ b/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java @@ -9,9 +9,9 @@ package org.opensearch.action.search; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java b/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java index 90cd1ed474d61..1a66311cd9a1b 100644 --- a/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java +++ b/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java @@ -11,8 +11,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java b/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java index 86c59bb4060a8..0aabd838171ba 100644 --- a/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java +++ b/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java @@ -10,21 +10,21 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; import java.util.List; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; /** * Response class for delete pits flow which clears the point in time search contexts diff --git a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java index b6ba0bee87932..2c6ab6437fd2a 100644 --- a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java index 7af687663833f..918583b3d510b 100644 --- a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java @@ -17,7 +17,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodeRequest.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodeRequest.java index de0c0dd9bbfc3..c0fe08793000c 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodeRequest.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodeRequest.java @@ -9,8 +9,8 @@ package org.opensearch.action.search; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java index fbde9b474107c..9bbc81c0d6a4c 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java @@ -11,8 +11,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java index b4ad2f6641087..948fe72eae817 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java @@ -11,8 +11,8 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java index ac1b703d37423..055eb84ab3811 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java @@ -12,8 +12,8 @@ import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java index 15fefb1671707..e9ff2fbf10f79 100644 --- a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java @@ -10,8 +10,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java index 4fd86febcb0ae..4211839ce6569 100644 --- a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java +++ b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java @@ -10,14 +10,14 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.pipeline.PipelineConfiguration; import java.io.IOException; @@ -27,7 +27,7 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * transport response for getting a search pipeline diff --git a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineTransportAction.java index 690990a0c4151..3b8c872e7a2f5 100644 --- a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineTransportAction.java @@ -17,7 +17,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/search/ListPitInfo.java b/server/src/main/java/org/opensearch/action/search/ListPitInfo.java index e46c2724ea651..e120507f4d47a 100644 --- a/server/src/main/java/org/opensearch/action/search/ListPitInfo.java +++ b/server/src/main/java/org/opensearch/action/search/ListPitInfo.java @@ -9,9 +9,9 @@ package org.opensearch.action.search; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java index d1095840189aa..fa8b5b9470723 100644 --- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java @@ -38,9 +38,9 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.IndicesOptions.WildcardStates; import org.opensearch.common.CheckedBiConsumer; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java b/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java index b6c30b6be00c9..c576d87d85b0f 100644 --- a/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java +++ b/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java @@ -37,9 +37,9 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/search/PitSearchContextIdForNode.java b/server/src/main/java/org/opensearch/action/search/PitSearchContextIdForNode.java index 577a559beb8f9..b885d3b351e80 100644 --- a/server/src/main/java/org/opensearch/action/search/PitSearchContextIdForNode.java +++ b/server/src/main/java/org/opensearch/action/search/PitSearchContextIdForNode.java @@ -8,9 +8,9 @@ package org.opensearch.action.search; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/search/PitService.java b/server/src/main/java/org/opensearch/action/search/PitService.java index ad9bd1273d898..336c0f2c49793 100644 --- a/server/src/main/java/org/opensearch/action/search/PitService.java +++ b/server/src/main/java/org/opensearch/action/search/PitService.java @@ -18,7 +18,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java index 178643641aa14..822d1f2320c9b 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java @@ -10,9 +10,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java index 2c0466dc939e7..da4a405c84555 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java @@ -22,7 +22,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.pipeline.SearchPipelineInfo; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java index 1aa18a695d31f..45c2dc4f29403 100644 --- a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.TopDocs; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/action/search/ReduceSearchPhaseException.java b/server/src/main/java/org/opensearch/action/search/ReduceSearchPhaseException.java index 57c1da5b25795..27522280336ea 100644 --- a/server/src/main/java/org/opensearch/action/search/ReduceSearchPhaseException.java +++ b/server/src/main/java/org/opensearch/action/search/ReduceSearchPhaseException.java @@ -32,7 +32,7 @@ package org.opensearch.action.search; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/search/SearchContextId.java b/server/src/main/java/org/opensearch/action/search/SearchContextId.java index 68c5f3dc49596..8c4c33d115355 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchContextId.java +++ b/server/src/main/java/org/opensearch/action/search/SearchContextId.java @@ -33,15 +33,15 @@ package org.opensearch.action.search; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.AliasFilter; diff --git a/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java index 7f218a3b1a17e..59c0e54bb6cbc 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java +++ b/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java @@ -33,9 +33,9 @@ package org.opensearch.action.search; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.internal.ShardSearchContextId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index 133f0a6be4628..c1df2fa913885 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -49,7 +49,7 @@ import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.collect.HppcMaps; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHit; diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseExecutionException.java index 65c876cbd7c61..b3ed42824e91a 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseExecutionException.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseExecutionException.java @@ -34,12 +34,12 @@ import org.opensearch.OpenSearchException; import org.opensearch.ExceptionsHelper; -import org.opensearch.action.ShardOperationFailedException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.action.ShardOperationFailedException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index 181c524054f91..43fec9e70ab88 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -38,8 +38,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponse.java b/server/src/main/java/org/opensearch/action/search/SearchResponse.java index a9f6e2bf66994..c7ab8f0858e7b 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponse.java @@ -36,9 +36,9 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.common.xcontent.XContentType; @@ -47,7 +47,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActions; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -65,7 +65,7 @@ import java.util.Objects; import java.util.function.Supplier; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A response of a search request. diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java index 20a7f576071b2..f90e98106f93f 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java @@ -41,7 +41,7 @@ import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.opensearch.OpenSearchException; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java index 7d152195627f1..214bc0448b90c 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java @@ -32,7 +32,7 @@ package org.opensearch.action.search; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.SearchHits; diff --git a/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java b/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java index 1625a09ac4e80..ddefb165f00b6 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/search/SearchShard.java b/server/src/main/java/org/opensearch/action/search/SearchShard.java index cdfc4e816b455..478788277bb19 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchShard.java +++ b/server/src/main/java/org/opensearch/action/search/SearchShard.java @@ -33,7 +33,7 @@ package org.opensearch.action.search; import org.opensearch.common.Nullable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Comparator; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java b/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java index 45e4c1a54eeba..fbd85a3fc0b8f 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java @@ -39,7 +39,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.Countable; import org.opensearch.common.util.PlainIterator; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; diff --git a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java index 13302f88fd87f..5a280818640ed 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java @@ -40,9 +40,9 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; diff --git a/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java index 891f7f3119541..50cd1dc2e2cd4 100644 --- a/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java @@ -35,23 +35,23 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.OriginalIndices; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchException; import org.opensearch.search.SearchShardTarget; import org.opensearch.transport.RemoteClusterAware; import java.io.IOException; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Represents a failure to search on a specific shard. diff --git a/server/src/main/java/org/opensearch/action/search/TransportClearScrollAction.java b/server/src/main/java/org/opensearch/action/search/TransportClearScrollAction.java index c9b6e967d153c..a44b80f13975c 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportClearScrollAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportClearScrollAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java b/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java index c6bf610edfb9a..c664eb9629216 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java @@ -15,11 +15,11 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java index 217fcc1489df7..6ad11aa4c6d59 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java @@ -12,7 +12,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java b/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java index 39299f9a33b18..561ca034da469 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java @@ -13,7 +13,7 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.SearchService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java index 58eb617a3d5fe..a3ee6e0a37e23 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index df2170cbe2af1..c4af9ffa20194 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -59,17 +59,17 @@ import org.opensearch.common.Nullable; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.Rewriteable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/opensearch/action/search/TransportSearchHelper.java index da432a73c8a1d..8c127983770b6 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchHelper.java @@ -33,8 +33,8 @@ package org.opensearch.action.search; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.search.SearchPhaseResult; diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchScrollAction.java index b67270ec9c9a5..ea29449582e7c 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchScrollAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchScrollAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/search/UpdatePitContextRequest.java b/server/src/main/java/org/opensearch/action/search/UpdatePitContextRequest.java index e6c9befb7938f..0d10390f1f660 100644 --- a/server/src/main/java/org/opensearch/action/search/UpdatePitContextRequest.java +++ b/server/src/main/java/org/opensearch/action/search/UpdatePitContextRequest.java @@ -8,8 +8,8 @@ package org.opensearch.action.search; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/action/search/UpdatePitContextResponse.java b/server/src/main/java/org/opensearch/action/search/UpdatePitContextResponse.java index 919dd87ea3041..da39aed20ef8e 100644 --- a/server/src/main/java/org/opensearch/action/search/UpdatePitContextResponse.java +++ b/server/src/main/java/org/opensearch/action/search/UpdatePitContextResponse.java @@ -8,8 +8,8 @@ package org.opensearch.action.search; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java b/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java index 10699690d0aff..3a687f946f08b 100644 --- a/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java +++ b/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java @@ -38,9 +38,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/HandledTransportAction.java b/server/src/main/java/org/opensearch/action/support/HandledTransportAction.java index 0bc46b21be3b8..7e030a34cdd91 100644 --- a/server/src/main/java/org/opensearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/opensearch/action/support/HandledTransportAction.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; diff --git a/server/src/main/java/org/opensearch/action/support/IndicesOptions.java b/server/src/main/java/org/opensearch/action/support/IndicesOptions.java index fe1b2efc05b80..d30ee7e11bdfa 100644 --- a/server/src/main/java/org/opensearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/opensearch/action/support/IndicesOptions.java @@ -33,8 +33,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/support/WriteRequest.java b/server/src/main/java/org/opensearch/action/support/WriteRequest.java index 16573ab619f2f..f462464b99ce8 100644 --- a/server/src/main/java/org/opensearch/action/support/WriteRequest.java +++ b/server/src/main/java/org/opensearch/action/support/WriteRequest.java @@ -37,9 +37,9 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.replication.ReplicatedWriteRequest; import org.opensearch.action.update.UpdateRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java index 84cb2965e6e61..ea9a77fc453f5 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastResponse.java b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastResponse.java index fbe19ad2bdcd1..96f7efd05b8ba 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastResponse.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastResponse.java @@ -33,20 +33,20 @@ package org.opensearch.action.support.broadcast; import org.opensearch.action.ActionResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActions; import java.io.IOException; import java.util.List; -import static org.opensearch.action.support.DefaultShardOperationFailedException.readShardOperationFailed; +import static org.opensearch.core.action.support.DefaultShardOperationFailedException.readShardOperationFailed; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardOperationFailedException.java b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardOperationFailedException.java index ed84b48f38387..03cdb1103343e 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardOperationFailedException.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardOperationFailedException.java @@ -34,8 +34,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardRequest.java b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardRequest.java index 02fc305d4eb0d..b426f6e41f81d 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardRequest.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardRequest.java @@ -35,9 +35,9 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.OriginalIndices; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardResponse.java b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardResponse.java index b3045d52450cc..39c524448bc5d 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardResponse.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastShardResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.support.broadcast; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index 5abf97b7ef979..cb77fa7c3ac57 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -50,8 +50,8 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 9e353a35831d0..bf71134ab7b88 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -38,7 +38,7 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.TransportActions; @@ -53,9 +53,9 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.NodeShouldNotConnectException; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java index e9fd0c77a5ec6..6dcc6ed1b098e 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java @@ -32,8 +32,8 @@ package org.opensearch.action.support.clustermanager; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java index 9d8a79cfed11d..affcb16327089 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java @@ -33,8 +33,8 @@ package org.opensearch.action.support.clustermanager; import org.opensearch.action.ActionRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java index 2f02a34526802..13c576bd120c7 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java @@ -56,8 +56,8 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterManagerThrottlingException; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; import org.opensearch.node.NodeClosedException; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeReadAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeReadAction.java index 1bfd7faa90262..e957142afd884 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeReadAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeReadAction.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/info/ClusterInfoRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/info/ClusterInfoRequest.java index 0059f5c836202..843a711dc3d65 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/info/ClusterInfoRequest.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/info/ClusterInfoRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java index c43256a61e8b4..7390a87de1f85 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java index 11f3eec5dfe37..1601dfd0c9893 100644 --- a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java +++ b/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java @@ -32,8 +32,8 @@ package org.opensearch.action.support.master; import org.opensearch.cluster.ack.AckedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java index 53e44266285d6..71695269396e1 100644 --- a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java +++ b/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/support/master/MasterNodeReadRequest.java b/server/src/main/java/org/opensearch/action/support/master/MasterNodeReadRequest.java index d8c407722ed8d..ea8f6a6e43cfe 100644 --- a/server/src/main/java/org/opensearch/action/support/master/MasterNodeReadRequest.java +++ b/server/src/main/java/org/opensearch/action/support/master/MasterNodeReadRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.support.master; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/master/MasterNodeRequest.java b/server/src/main/java/org/opensearch/action/support/master/MasterNodeRequest.java index fb86742186c9c..cfab63a845f7f 100644 --- a/server/src/main/java/org/opensearch/action/support/master/MasterNodeRequest.java +++ b/server/src/main/java/org/opensearch/action/support/master/MasterNodeRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.support.master; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java b/server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java index 0a2f93d4e8caa..fd54e810528d3 100644 --- a/server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java +++ b/server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.support.master; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java index 2524e839ba11e..53a597214256d 100644 --- a/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeReadAction.java b/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeReadAction.java index 0b3f309acc189..8adfba6c8ee02 100644 --- a/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeReadAction.java +++ b/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeReadAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/opensearch/action/support/master/info/ClusterInfoRequest.java index 1f59fca1b4081..0b66e3d932603 100644 --- a/server/src/main/java/org/opensearch/action/support/master/info/ClusterInfoRequest.java +++ b/server/src/main/java/org/opensearch/action/support/master/info/ClusterInfoRequest.java @@ -32,7 +32,7 @@ package org.opensearch.action.support.master.info; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java b/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java index 26d31b874f2c0..8ba6653892f88 100644 --- a/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java @@ -35,7 +35,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java index 62e814d7e7ec0..4a94f790c3443 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.support.nodes; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java index f4b25a43dc206..609b58322b990 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesResponse.java b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesResponse.java index a47eb4be898ca..bf68b5201c3f8 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesResponse.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesResponse.java @@ -35,8 +35,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.action.FailedNodeException; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java index a12e9b753599d..3f39d7b786ccb 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java @@ -41,8 +41,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.NodeShouldNotConnectException; diff --git a/server/src/main/java/org/opensearch/action/support/replication/BasicReplicationRequest.java b/server/src/main/java/org/opensearch/action/support/replication/BasicReplicationRequest.java index bd0663fc3d352..b2a53fec5c281 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/BasicReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/support/replication/BasicReplicationRequest.java @@ -32,8 +32,8 @@ package org.opensearch.action.support.replication; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java b/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java index c3b7ec9a03967..d2c4e7563195f 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java +++ b/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java @@ -38,7 +38,7 @@ import org.opensearch.index.shard.PrimaryShardClosedException; import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicatedWriteRequest.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicatedWriteRequest.java index 8018de175b47e..55773c8721c2a 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicatedWriteRequest.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicatedWriteRequest.java @@ -37,9 +37,9 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.WriteRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java index a7c7a799883a6..1affc9202c32b 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java @@ -48,14 +48,14 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.NodeClosedException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationRequest.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationRequest.java index 72214e73b9180..d5a2d37daa504 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationRequest.java @@ -40,10 +40,10 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java index 6f8f93ccc0ce3..1f0e61a134eb2 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java @@ -35,24 +35,24 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionResponse; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Base class for write action responses. diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationTask.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationTask.java index 7d95beb2eabc3..5aa37ec65ff0d 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationTask.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationTask.java @@ -33,8 +33,8 @@ package org.opensearch.action.support.replication; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java index b4bacefad8ad2..41f05ade7fdf0 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -36,7 +36,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TransportActions; import org.opensearch.action.support.broadcast.BroadcastRequest; @@ -47,9 +47,9 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.CountDown; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java index 3feb5b23e5540..5b3be4d077f3c 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java @@ -59,9 +59,9 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -75,7 +75,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.indices.IndexClosedException; diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java index 44e2f34b17377..a47d986c74c1e 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.index.IndexingPressureService; @@ -54,7 +54,7 @@ import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.shard.PrimaryShardClosedException; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.Translog.Location; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/InstanceShardOperationRequest.java b/server/src/main/java/org/opensearch/action/support/single/instance/InstanceShardOperationRequest.java index 85c15e1a01921..7a722ab2a5a35 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/InstanceShardOperationRequest.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/InstanceShardOperationRequest.java @@ -38,10 +38,10 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.concurrent.TimeUnit; diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index bb9d685deedcd..8014f1b280bf2 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -47,12 +47,12 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java b/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java index 61777e2b7bae2..c474096ff94e4 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java @@ -38,9 +38,9 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java index 39398392c0ee8..9a5ce608da472 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java @@ -52,10 +52,10 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.logging.LoggerMessageFormat; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; diff --git a/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java b/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java index 9172bb0aca098..43ece159247bc 100644 --- a/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java +++ b/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CollectionUtils; diff --git a/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksResponse.java b/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksResponse.java index fd90f9b389819..3bd126b452f5c 100644 --- a/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksResponse.java +++ b/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksResponse.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.action.FailedNodeException; import org.opensearch.action.TaskOperationFailure; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java index e2e872eea5dc0..bfd207e6f969f 100644 --- a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java @@ -45,9 +45,9 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java index 0367c01d7e4b7..80ca1629417ad 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java @@ -32,9 +32,9 @@ package org.opensearch.action.termvectors; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java index 44c6ca58c3a25..c055564c3fcbe 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java @@ -39,8 +39,8 @@ import org.opensearch.action.RealtimeRequest; import org.opensearch.action.ValidateActions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java index ff84ceb22e043..cc482b5190e32 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java @@ -35,9 +35,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java index bf2d4c676f409..a31e3085cbc17 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.single.shard.SingleShardRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java index 806510f58f03a..05d949df8db00 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java +++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsShardResponse.java @@ -33,8 +33,8 @@ package org.opensearch.action.termvectors; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java index 0b2cd3bb2509f..ff7a8de3a97d3 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java @@ -46,8 +46,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.RamUsageEstimator; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java index 2a170dec776a4..d33882fc1d53d 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java @@ -41,10 +41,10 @@ import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.MediaType; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java index 834afd1a4a952..8889d72086f47 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java @@ -42,11 +42,11 @@ import org.apache.lucene.util.CharsRefBuilder; import org.opensearch.Version; import org.opensearch.action.ActionResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsWriter.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsWriter.java index e7772cf300f5a..0e16291131e2a 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsWriter.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsWriter.java @@ -40,7 +40,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.action.termvectors.TermVectorsRequest.Flag; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java index d90c3968bddbb..4f3d0f2a095df 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java @@ -43,7 +43,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/server/src/main/java/org/opensearch/action/termvectors/TransportShardMultiTermsVectorAction.java index b5928dc99571f..73da857754cfa 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TransportShardMultiTermsVectorAction.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TransportShardMultiTermsVectorAction.java @@ -41,10 +41,10 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.termvectors.TermVectorsService; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java index 11770cd10d982..af6b0e6b8e251 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java @@ -42,10 +42,10 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.termvectors.TermVectorsService; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java index e86cfa70f1169..d20d5e229a4c7 100644 --- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java @@ -55,18 +55,18 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NotSerializableExceptionWrapper; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java index 47a764f9effa3..e3aa4f1161ac7 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java @@ -40,10 +40,10 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Requests; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -53,7 +53,7 @@ import org.opensearch.index.get.GetResult; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.script.Script; import org.opensearch.script.ScriptService; import org.opensearch.script.UpdateScript; diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java index 315ec47e30296..8f529ec855d9b 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java @@ -42,8 +42,8 @@ import org.opensearch.action.support.replication.ReplicationRequest; import org.opensearch.action.support.single.instance.InstanceShardOperationRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; @@ -58,7 +58,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.fetch.subphase.FetchSourceContext; diff --git a/server/src/main/java/org/opensearch/action/update/UpdateResponse.java b/server/src/main/java/org/opensearch/action/update/UpdateResponse.java index c29a80157dfe0..61767abb99cb9 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateResponse.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateResponse.java @@ -33,17 +33,17 @@ package org.opensearch.action.update; import org.opensearch.action.DocWriteResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.get.GetResult; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Transport response for updating an index diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 25b6c002afc47..2a23b501a8a0e 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -53,7 +53,7 @@ import org.opensearch.common.network.IfConfig; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/main/java/org/opensearch/cli/KeyStoreAwareCommand.java b/server/src/main/java/org/opensearch/cli/KeyStoreAwareCommand.java index 17067909315b2..6cd266252b369 100644 --- a/server/src/main/java/org/opensearch/cli/KeyStoreAwareCommand.java +++ b/server/src/main/java/org/opensearch/cli/KeyStoreAwareCommand.java @@ -34,7 +34,7 @@ import joptsimple.OptionSet; import org.opensearch.common.settings.KeyStoreWrapper; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java index 153485c36d77c..f3c04b23dfd54 100644 --- a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java @@ -158,7 +158,7 @@ import org.opensearch.action.search.GetSearchPipelineResponse; import org.opensearch.action.search.PutSearchPipelineRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/client/node/NodeClient.java b/server/src/main/java/org/opensearch/client/node/NodeClient.java index 3341bfe326990..60551ade09416 100644 --- a/server/src/main/java/org/opensearch/client/node/NodeClient.java +++ b/server/src/main/java/org/opensearch/client/node/NodeClient.java @@ -41,7 +41,7 @@ import org.opensearch.client.Client; import org.opensearch.client.support.AbstractClient; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index c67b4493ba4b9..40489e29ed9b5 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -410,7 +410,7 @@ import org.opensearch.client.OpenSearchClient; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/client/transport/NoNodeAvailableException.java b/server/src/main/java/org/opensearch/client/transport/NoNodeAvailableException.java index e3424689c5152..75dbd90275367 100644 --- a/server/src/main/java/org/opensearch/client/transport/NoNodeAvailableException.java +++ b/server/src/main/java/org/opensearch/client/transport/NoNodeAvailableException.java @@ -33,8 +33,8 @@ package org.opensearch.client.transport; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java index 834d78a384423..74af3472433ba 100644 --- a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java +++ b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java @@ -33,8 +33,8 @@ package org.opensearch.cluster; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/AbstractNamedDiffable.java b/server/src/main/java/org/opensearch/cluster/AbstractNamedDiffable.java index f4780eb71edda..49837aea83708 100644 --- a/server/src/main/java/org/opensearch/cluster/AbstractNamedDiffable.java +++ b/server/src/main/java/org/opensearch/cluster/AbstractNamedDiffable.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java b/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java index 28085dd6e3860..236da42f94506 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.gateway.GatewayService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.ArrayList; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java index ffa3d0d19fb71..cac98a262df53 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java @@ -36,13 +36,13 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.Version; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.remote.filecache.FileCacheStats; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index cf00f44bea4b4..b80fd1d746831 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -79,9 +79,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.ParseField; import org.opensearch.common.inject.AbstractModule; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/cluster/ClusterName.java b/server/src/main/java/org/opensearch/cluster/ClusterName.java index 77fc21b739f1a..440fde284afb8 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterName.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterName.java @@ -32,9 +32,9 @@ package org.opensearch.cluster; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/cluster/ClusterSettingsResponse.java b/server/src/main/java/org/opensearch/cluster/ClusterSettingsResponse.java index e84c2c902abdd..ee5c8c00dfaf4 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterSettingsResponse.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterSettingsResponse.java @@ -9,8 +9,8 @@ package org.opensearch.cluster; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.transport.TransportResponse; diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index e04ce938e7fdc..bf4d7977222ab 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -49,13 +49,13 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.VersionedNamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VersionedNamedWriteable; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/cluster/Diff.java b/server/src/main/java/org/opensearch/cluster/Diff.java index 9af6afed0d13b..c0e8e7038d9b4 100644 --- a/server/src/main/java/org/opensearch/cluster/Diff.java +++ b/server/src/main/java/org/opensearch/cluster/Diff.java @@ -32,7 +32,7 @@ package org.opensearch.cluster; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Represents difference between states of cluster state parts diff --git a/server/src/main/java/org/opensearch/cluster/Diffable.java b/server/src/main/java/org/opensearch/cluster/Diffable.java index 23c30527befa9..a3fedf4903089 100644 --- a/server/src/main/java/org/opensearch/cluster/Diffable.java +++ b/server/src/main/java/org/opensearch/cluster/Diffable.java @@ -32,7 +32,7 @@ package org.opensearch.cluster; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Cluster state part, changes in which can be serialized diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index e3ecf48757af5..dd2232968114e 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -33,9 +33,9 @@ package org.opensearch.cluster; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable.Reader; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/cluster/DiskUsage.java b/server/src/main/java/org/opensearch/cluster/DiskUsage.java index 84c7930beac78..961bfce053243 100644 --- a/server/src/main/java/org/opensearch/cluster/DiskUsage.java +++ b/server/src/main/java/org/opensearch/cluster/DiskUsage.java @@ -33,9 +33,9 @@ package org.opensearch.cluster; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/IncompatibleClusterStateVersionException.java b/server/src/main/java/org/opensearch/cluster/IncompatibleClusterStateVersionException.java index 8dd988b0fba5d..671748e52c544 100644 --- a/server/src/main/java/org/opensearch/cluster/IncompatibleClusterStateVersionException.java +++ b/server/src/main/java/org/opensearch/cluster/IncompatibleClusterStateVersionException.java @@ -33,7 +33,7 @@ package org.opensearch.cluster; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/NamedDiff.java b/server/src/main/java/org/opensearch/cluster/NamedDiff.java index 2b8dfa14f9ea6..ce971aa723394 100644 --- a/server/src/main/java/org/opensearch/cluster/NamedDiff.java +++ b/server/src/main/java/org/opensearch/cluster/NamedDiff.java @@ -33,7 +33,7 @@ package org.opensearch.cluster; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; /** * Diff that also support NamedWriteable interface diff --git a/server/src/main/java/org/opensearch/cluster/NamedDiffable.java b/server/src/main/java/org/opensearch/cluster/NamedDiffable.java index b47e180cf583e..ee73666d8b3b2 100644 --- a/server/src/main/java/org/opensearch/cluster/NamedDiffable.java +++ b/server/src/main/java/org/opensearch/cluster/NamedDiffable.java @@ -32,7 +32,7 @@ package org.opensearch.cluster; -import org.opensearch.common.io.stream.VersionedNamedWriteable; +import org.opensearch.core.common.io.stream.VersionedNamedWriteable; /** * Diff that also support {@link VersionedNamedWriteable} interface diff --git a/server/src/main/java/org/opensearch/cluster/NamedDiffableValueSerializer.java b/server/src/main/java/org/opensearch/cluster/NamedDiffableValueSerializer.java index ecf42e413459c..f5ee364078361 100644 --- a/server/src/main/java/org/opensearch/cluster/NamedDiffableValueSerializer.java +++ b/server/src/main/java/org/opensearch/cluster/NamedDiffableValueSerializer.java @@ -33,7 +33,7 @@ package org.opensearch.cluster; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/NotClusterManagerException.java b/server/src/main/java/org/opensearch/cluster/NotClusterManagerException.java index 522bad0f9d682..d96bb521f399e 100644 --- a/server/src/main/java/org/opensearch/cluster/NotClusterManagerException.java +++ b/server/src/main/java/org/opensearch/cluster/NotClusterManagerException.java @@ -32,7 +32,7 @@ package org.opensearch.cluster; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/NotMasterException.java b/server/src/main/java/org/opensearch/cluster/NotMasterException.java index 1dbf13212d2dd..8cdd0f8332212 100644 --- a/server/src/main/java/org/opensearch/cluster/NotMasterException.java +++ b/server/src/main/java/org/opensearch/cluster/NotMasterException.java @@ -31,7 +31,7 @@ package org.opensearch.cluster; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java index 601fa02eea25b..c399bd59dbbe1 100644 --- a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java @@ -34,9 +34,9 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.repositories.RepositoryOperation; diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java index 8fa56fd073ac0..ce7e4f6d918be 100644 --- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java @@ -34,12 +34,12 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.Snapshot; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index 6319cf6d56979..1108a595cd5ac 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -35,9 +35,9 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.repositories.RepositoryOperation; diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index ffc99c34fcac5..855f7755419d8 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -36,14 +36,14 @@ import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoryOperation; import org.opensearch.repositories.RepositoryShardId; diff --git a/server/src/main/java/org/opensearch/cluster/ack/IndicesClusterStateUpdateRequest.java b/server/src/main/java/org/opensearch/cluster/ack/IndicesClusterStateUpdateRequest.java index c235b132d8f71..d9753027ffc67 100644 --- a/server/src/main/java/org/opensearch/cluster/ack/IndicesClusterStateUpdateRequest.java +++ b/server/src/main/java/org/opensearch/cluster/ack/IndicesClusterStateUpdateRequest.java @@ -31,7 +31,7 @@ package org.opensearch.cluster.ack; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; /** * Base cluster state update request that allows to execute update against multiple indices diff --git a/server/src/main/java/org/opensearch/cluster/action/index/MappingUpdatedAction.java b/server/src/main/java/org/opensearch/cluster/action/index/MappingUpdatedAction.java index 4bb4137f9d7a2..c589d9bfeeab2 100644 --- a/server/src/main/java/org/opensearch/cluster/action/index/MappingUpdatedAction.java +++ b/server/src/main/java/org/opensearch/cluster/action/index/MappingUpdatedAction.java @@ -49,7 +49,7 @@ import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.common.util.concurrent.UncategorizedExecutionException; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.Mapping; import java.util.concurrent.Semaphore; diff --git a/server/src/main/java/org/opensearch/cluster/action/index/NodeMappingRefreshAction.java b/server/src/main/java/org/opensearch/cluster/action/index/NodeMappingRefreshAction.java index 5d2f9a5957772..e6781fc22e1a7 100644 --- a/server/src/main/java/org/opensearch/cluster/action/index/NodeMappingRefreshAction.java +++ b/server/src/main/java/org/opensearch/cluster/action/index/NodeMappingRefreshAction.java @@ -40,8 +40,8 @@ import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.EmptyTransportResponseHandler; diff --git a/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java index 002c5fd3b89db..9d9b6c52f6b25 100644 --- a/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java @@ -58,11 +58,11 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java index 7fb5c03ffb35a..75b73be8fa12e 100644 --- a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java +++ b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java @@ -15,9 +15,9 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.WeightedRouting; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -27,7 +27,7 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Cluster Awareness AttributeValue Health information diff --git a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java index 26c4f75d446e6..340fcfe0d0d31 100644 --- a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java +++ b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java @@ -12,9 +12,9 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -29,7 +29,7 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Cluster Awareness health information diff --git a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java index 5938d345d28ab..85c5ee60e3ba5 100644 --- a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java +++ b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java @@ -11,9 +11,9 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java index d9fe2bd29522f..a839644f74cb3 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java @@ -33,12 +33,12 @@ package org.opensearch.cluster.block; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.EnumSet; diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java index c507b76168e82..4673f075e8439 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java @@ -33,9 +33,9 @@ package org.opensearch.cluster.block; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index cf77349271eb5..33d55e6e747a3 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -37,11 +37,11 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MetadataIndexStateService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.set.Sets; import org.opensearch.index.IndexModule; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java b/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java index 3ca0ef5ade225..729d76c72e99e 100644 --- a/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java +++ b/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java @@ -8,7 +8,7 @@ package org.opensearch.cluster.block; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.util.Set; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/ApplyCommitRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/ApplyCommitRequest.java index 3536a11f9e2e0..a722ccd5c6aff 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/ApplyCommitRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/ApplyCommitRequest.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CompressedStreamUtils.java b/server/src/main/java/org/opensearch/cluster/coordination/CompressedStreamUtils.java index 97a93a8fa99d4..ab3cb51683c8f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CompressedStreamUtils.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CompressedStreamUtils.java @@ -12,16 +12,16 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.Compressor; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.BytesTransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 459d065a75490..520eb0e15754d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -33,9 +33,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationStateRejectedException.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationStateRejectedException.java index ece8f3e6d7345..8dc8d4295072a 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationStateRejectedException.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationStateRejectedException.java @@ -33,7 +33,7 @@ package org.opensearch.cluster.coordination; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 546fe2d4643d3..de751d881bc0e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -63,7 +63,7 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; import org.opensearch.common.component.AbstractLifecycleComponent; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FailedToCommitClusterStateException.java b/server/src/main/java/org/opensearch/cluster/coordination/FailedToCommitClusterStateException.java index db431a1a018c8..8379bcaa0f0d8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/FailedToCommitClusterStateException.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/FailedToCommitClusterStateException.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.coordination; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java index e69a78eae041d..08008152cfcd6 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.coordination.Coordinator.Mode; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Join.java b/server/src/main/java/org/opensearch/cluster/coordination/Join.java index d039ccc8a1127..58fa85992ebc8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Join.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Join.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 114051311ab97..42f09f95a7f56 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -50,10 +50,10 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.common.Priority; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java index 76e56bf2849cd..04f87d16ee400 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java index 06b722069e27a..f43abf0080575 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java b/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java index 71e9a87cdffae..b377fe592b0f4 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java @@ -37,7 +37,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.EnumSet; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/NodeHealthCheckFailureException.java b/server/src/main/java/org/opensearch/cluster/coordination/NodeHealthCheckFailureException.java index d0919aa9e8686..43cc975573072 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/NodeHealthCheckFailureException.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/NodeHealthCheckFailureException.java @@ -33,7 +33,7 @@ package org.opensearch.cluster.coordination; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/OpenSearchNodeCommand.java b/server/src/main/java/org/opensearch/cluster/coordination/OpenSearchNodeCommand.java index 25b1334949a9c..3f4c4ee6f180a 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/OpenSearchNodeCommand.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/OpenSearchNodeCommand.java @@ -50,7 +50,7 @@ import org.opensearch.cluster.metadata.DataStreamMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PeersResponse.java b/server/src/main/java/org/opensearch/cluster/coordination/PeersResponse.java index 0b0d6f8e57174..b68f689ef63fd 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PeersResponse.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PeersResponse.java @@ -33,8 +33,8 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PendingClusterStateStats.java b/server/src/main/java/org/opensearch/cluster/coordination/PendingClusterStateStats.java index b431c638be049..9cab59c5fa492 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PendingClusterStateStats.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PendingClusterStateStats.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.coordination; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java index 2cf7ebcc850a5..f8e568d3f5a0a 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lease.Releasable; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteRequest.java index c133a51239335..85632bdc06e75 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteRequest.java @@ -33,8 +33,8 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteResponse.java b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteResponse.java index f18726c8af229..c8186441db449 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteResponse.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteResponse.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.coordination; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 21ef89e9d5790..60c931a601561 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -43,9 +43,9 @@ import org.opensearch.cluster.IncompatibleClusterStateVersionException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.TransportChannel; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublishClusterStateStats.java b/server/src/main/java/org/opensearch/cluster/coordination/PublishClusterStateStats.java index b0c9b2bb0e21c..6a1dca0d27208 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublishClusterStateStats.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublishClusterStateStats.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.coordination; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublishResponse.java b/server/src/main/java/org/opensearch/cluster/coordination/PublishResponse.java index 5d809617019ac..23a60cb88283e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublishResponse.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublishResponse.java @@ -31,9 +31,9 @@ package org.opensearch.cluster.coordination; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublishWithJoinResponse.java b/server/src/main/java/org/opensearch/cluster/coordination/PublishWithJoinResponse.java index f02f917e67800..f99ba82be5514 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublishWithJoinResponse.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublishWithJoinResponse.java @@ -31,8 +31,8 @@ package org.opensearch.cluster.coordination; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java index 09ebcd7fd074e..de58eb721b28f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/TermVersionRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/TermVersionRequest.java index 0b210ef21f45b..81923f4211b7e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/TermVersionRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/TermVersionRequest.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/ValidateJoinRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/ValidateJoinRequest.java index 3f8c00da9a76c..34bbfc09d92dd 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/ValidateJoinRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/ValidateJoinRequest.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.ClusterState; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java index bf2487a1a0e18..3f67870781580 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java @@ -8,9 +8,9 @@ package org.opensearch.cluster.decommission; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java index 0a4b3fe8fde3c..8af783bbdc52e 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java @@ -15,8 +15,8 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java index 1b605643d85fe..7a21dae35af94 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java @@ -27,7 +27,7 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.http.HttpStats; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissioningFailedException.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissioningFailedException.java index 9d1325ccf4912..6f3b319a543c6 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissioningFailedException.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissioningFailedException.java @@ -9,9 +9,9 @@ package org.opensearch.cluster.decommission; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/decommission/NodeDecommissionedException.java b/server/src/main/java/org/opensearch/cluster/decommission/NodeDecommissionedException.java index c91509a0db161..ef85335be9a74 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/NodeDecommissionedException.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/NodeDecommissionedException.java @@ -9,8 +9,8 @@ package org.opensearch.cluster.decommission; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java b/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java index 4daf0cb6adbb2..5ea482b8b8ffa 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.health; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java index f9a244e5c1909..0bb762e3ff744 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; @@ -56,7 +56,7 @@ import static java.util.Collections.emptyMap; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Cluster Index Health Information diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java index a70492e20aa6c..00a83c85c17be 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java @@ -38,9 +38,9 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.UnassignedInfo.AllocationStatus; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; @@ -52,7 +52,7 @@ import java.util.Locale; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; /** diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java index 981015c7dc7dc..083159bffdc2b 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java @@ -35,10 +35,10 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java index 72b6380e78a1e..44f6f2d7313a3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java @@ -36,10 +36,10 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java index 948f756331d46..5a019804f5eac 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java index 51f1e75636273..52096422248a5 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java index 4fd7dcf4f5d4c..ce806b2aa1f12 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplateMetadata.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.NamedDiff; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java index e304e4da6af08..15e5cb5873719 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java @@ -37,9 +37,9 @@ import org.opensearch.cluster.metadata.DataStream.TimestampField; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadata.java index 89e412cee748b..d34416c70dc16 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.NamedDiff; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.ParseField; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java b/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java index f4be1cfff489c..76e1a0197bc1c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java @@ -38,14 +38,14 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/DataStreamMetadata.java index 343857c54b17c..89fe6e9be2320 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DataStreamMetadata.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.NamedDiff; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index 2372ec75445c9..8209c7bb56ad8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -34,8 +34,8 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.Diffable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.AbstractMap; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java index 5eaebf2f046db..5bf4827cbbe46 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.NamedDiff; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; @@ -47,7 +47,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.time.Instant; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index ddc603541e038..b6572b15a48c9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -48,9 +48,9 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; @@ -62,12 +62,12 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.MetadataStateFormat; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.time.Instant; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java index 4c26a03627270..52a76e9dd90f1 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java @@ -48,7 +48,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.InvalidIndexNameException; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index d422122356b04..e430ba5465499 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -38,8 +38,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Manifest.java b/server/src/main/java/org/opensearch/cluster/metadata/Manifest.java index 28810174c558c..695b8756b543c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Manifest.java @@ -40,7 +40,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.MetadataStateFormat; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java index 23d2a1667afaf..c77d2d6b8b2b3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java @@ -35,10 +35,10 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 5d0e4f9aa7e3f..3e36a5e65bf07 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -49,8 +49,8 @@ import org.opensearch.cluster.decommission.DecommissionAttributeMetadata; import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -63,10 +63,10 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.MetadataStateFormat; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.plugins.MapperPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java index 65cc775662831..91ff4f7652879 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -54,7 +54,7 @@ import org.opensearch.core.xcontent.ObjectPath; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MetadataFieldMapper; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 3fff6e8823e9e..97798f9627c60 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -76,7 +76,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataDeleteIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataDeleteIndexService.java index b23784bcd18aa..4ee66a9920898 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataDeleteIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataDeleteIndexService.java @@ -50,7 +50,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.snapshots.RestoreService; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotsService; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexAliasesService.java index 83e2511574867..19d2f4eccd7b0 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexAliasesService.java @@ -46,7 +46,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java index fad2d40cd9ff6..199ea21119aeb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java @@ -80,12 +80,12 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.RestoreService; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotsService; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java index 76860e568f1a1..4f12313f951bc 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java @@ -50,7 +50,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.UUIDs; import org.opensearch.common.ValidationException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.inject.Inject; import org.opensearch.common.logging.HeaderWarning; @@ -64,7 +64,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java index 9cb1eb6576ac5..845438b2363c3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java @@ -53,7 +53,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index 2b5e236bef8c9..a5caf3269ef26 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -58,7 +58,7 @@ import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ProcessClusterEventTimeoutException.java b/server/src/main/java/org/opensearch/cluster/metadata/ProcessClusterEventTimeoutException.java index cda032aa4628f..e3ef553540093 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ProcessClusterEventTimeoutException.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ProcessClusterEventTimeoutException.java @@ -33,9 +33,9 @@ package org.opensearch.cluster.metadata; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 06b97fdd31848..e654745e8ccdb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java index db4e5d8137a20..c9ed07cc9c2a8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java @@ -31,9 +31,9 @@ package org.opensearch.cluster.metadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.repositories.RepositoryData; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Template.java b/server/src/main/java/org/opensearch/cluster/metadata/Template.java index 01110d0402e66..45d11dd9250e8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Template.java @@ -37,8 +37,8 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java index b7c9c1512956d..c8c28c5db67c5 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.ClusterStateListener; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/WeightedRoutingMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/WeightedRoutingMetadata.java index 8ac8ec00a010d..6cfef67f2ceba 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/WeightedRoutingMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/WeightedRoutingMetadata.java @@ -16,8 +16,8 @@ import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.routing.WeightedRouting; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index ea9d9e30676bf..d6ba0199d193c 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index 454725704ba94..060c7c5eb8d1a 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.regex.Regex; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.set.Sets; diff --git a/server/src/main/java/org/opensearch/cluster/routing/AllocationId.java b/server/src/main/java/org/opensearch/cluster/routing/AllocationId.java index b7414ad26f62a..0bc434090b719 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/AllocationId.java +++ b/server/src/main/java/org/opensearch/cluster/routing/AllocationId.java @@ -35,9 +35,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java b/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java index 72c189f20eaf6..1724b26f5475d 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java @@ -14,8 +14,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.search.SearchShardIterator; import org.opensearch.cluster.ClusterState; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchShardTarget; import java.util.List; diff --git a/server/src/main/java/org/opensearch/cluster/routing/IllegalShardRoutingStateException.java b/server/src/main/java/org/opensearch/cluster/routing/IllegalShardRoutingStateException.java index 96fc2f4cf4ed6..27acbcde04d62 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IllegalShardRoutingStateException.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IllegalShardRoutingStateException.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.routing; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 9463f9ff0a422..224cd53126145 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -48,10 +48,10 @@ import org.opensearch.cluster.routing.RecoverySource.RemoteStoreRecoverySource; import org.opensearch.common.Randomness; import org.opensearch.common.collect.ImmutableOpenIntMap; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 7befec56abaa6..f9f57eeda4e98 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -40,11 +40,11 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Randomness; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.ResponseCollectorService; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/NodeWeighedAwayException.java b/server/src/main/java/org/opensearch/cluster/routing/NodeWeighedAwayException.java index ea614145a61bd..4fc0da07ff447 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/NodeWeighedAwayException.java +++ b/server/src/main/java/org/opensearch/cluster/routing/NodeWeighedAwayException.java @@ -9,8 +9,8 @@ package org.opensearch.cluster.routing; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index ade2cda797334..24e035e3db643 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -44,7 +44,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.ResponseCollectorService; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/cluster/routing/PlainShardIterator.java b/server/src/main/java/org/opensearch/cluster/routing/PlainShardIterator.java index 7b47fa6cb519d..b91eb96819676 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/PlainShardIterator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/PlainShardIterator.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.routing; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.List; diff --git a/server/src/main/java/org/opensearch/cluster/routing/PreferenceBasedSearchNotAllowedException.java b/server/src/main/java/org/opensearch/cluster/routing/PreferenceBasedSearchNotAllowedException.java index 2cdaeee7e90f2..308520a870389 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/PreferenceBasedSearchNotAllowedException.java +++ b/server/src/main/java/org/opensearch/cluster/routing/PreferenceBasedSearchNotAllowedException.java @@ -9,8 +9,8 @@ package org.opensearch.cluster.routing; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index 41e06ac2a8b24..aafc37fba9e85 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -35,9 +35,9 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingException.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingException.java index a45656ada929f..8db6a21958d4b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingException.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingException.java @@ -33,7 +33,7 @@ package org.opensearch.cluster.routing; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java index 413ddff72f7a5..e10734eb17e0d 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java @@ -35,8 +35,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.ArrayList; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java index e67c37d15d18e..dc9973a7ac005 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java @@ -45,8 +45,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Randomness; import org.opensearch.common.collect.Tuple; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 065c2e02b1926..23631f438b418 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -41,12 +41,12 @@ import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.cluster.routing.RecoverySource.RemoteStoreRecoverySource; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.iterable.Iterables; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/ShardIterator.java b/server/src/main/java/org/opensearch/cluster/routing/ShardIterator.java index 12c531d5fd767..623846b6e1dda 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/ShardIterator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/ShardIterator.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.routing; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /** * Allows to iterate over a set of shard instances (routing) within a shard id group. diff --git a/server/src/main/java/org/opensearch/cluster/routing/ShardRouting.java b/server/src/main/java/org/opensearch/cluster/routing/ShardRouting.java index 4c1a0d8539b1f..aa893ec735bac 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/ShardRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/ShardRouting.java @@ -36,13 +36,13 @@ import org.opensearch.cluster.routing.RecoverySource.PeerRecoverySource; import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java index 45fdf7ca641de..a4ecbadc34702 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java +++ b/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java @@ -38,9 +38,9 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/cluster/routing/UnsupportedWeightedRoutingStateException.java b/server/src/main/java/org/opensearch/cluster/routing/UnsupportedWeightedRoutingStateException.java index fd4fd4163ede6..451084fb79100 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/UnsupportedWeightedRoutingStateException.java +++ b/server/src/main/java/org/opensearch/cluster/routing/UnsupportedWeightedRoutingStateException.java @@ -9,8 +9,8 @@ package org.opensearch.cluster.routing; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java index 8dcf9d8dc6bd7..01471ab664294 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java @@ -8,9 +8,9 @@ package org.opensearch.cluster.routing; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java index 6b5648c58c2cd..49c6ceee74996 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java +++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java @@ -8,9 +8,9 @@ package org.opensearch.cluster.routing; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java index 1d4114618a0cb..59a39b358cb70 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java @@ -35,9 +35,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.allocation.decider.Decision.Type; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocateUnassignedDecision.java index 5db69a3845f9a..627d71522f9ed 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocateUnassignedDecision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocateUnassignedDecision.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.cluster.routing.allocation.decider.Decision.Type; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationDecision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationDecision.java index d84a31db6244c..3f1c025fce405 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationDecision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationDecision.java @@ -34,9 +34,9 @@ import org.opensearch.cluster.routing.UnassignedInfo.AllocationStatus; import org.opensearch.cluster.routing.allocation.decider.Decision; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java index b14f0f8eb3c35..ae8d92dae6811 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java @@ -43,8 +43,8 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.Collections; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/MoveDecision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/MoveDecision.java index 4ed638aae4cf3..fc39f813ca181 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/MoveDecision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/MoveDecision.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.cluster.routing.allocation.decider.Decision.Type; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java index d50a474838a6b..946e2d5e5f3dc 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java index 7cdec3b3f48f2..6c84957d6a788 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java @@ -34,8 +34,8 @@ import org.opensearch.cluster.routing.allocation.command.AllocationCommand; import org.opensearch.cluster.routing.allocation.decider.Decision; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java index 6dae12a592a6e..9e8e399384467 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.Decision; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.RestoreService.RestoreInProgressUpdater; import org.opensearch.snapshots.SnapshotShardSizeInfo; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java index ff75d0bc8e512..769212703b48b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java @@ -33,8 +33,8 @@ package org.opensearch.cluster.routing.allocation; import org.opensearch.cluster.routing.allocation.decider.Decision; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/ShardAllocationDecision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/ShardAllocationDecision.java index df4080d1a2276..e1a7fb43827f3 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/ShardAllocationDecision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/ShardAllocationDecision.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.routing.allocation; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/StaleShard.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/StaleShard.java index c442c4212928d..b5511f4a177e8 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/StaleShard.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/StaleShard.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.routing.allocation; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /** * A class that represents a stale shard copy. diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java index 072073cd6f337..e8e6b48d0901e 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java @@ -43,8 +43,8 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java index b49377492e381..f5093ef4d9243 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java @@ -43,11 +43,11 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java index 72e09466fc37a..d545c9090e81f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java index 6822637480760..983265ed5f960 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java index 444657c691be7..38a7d83ccc7b5 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java @@ -35,7 +35,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.cluster.routing.allocation.RerouteExplanation; import org.opensearch.cluster.routing.allocation.RoutingAllocation; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.common.network.NetworkModule; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommands.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommands.java index 5960179aaa110..e72fe26def31b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommands.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommands.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.RoutingExplanations; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java index 34c5d1103cbd2..17c538c60fa90 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java @@ -33,8 +33,8 @@ package org.opensearch.cluster.routing.allocation.command; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/CancelAllocationCommand.java index 538393d9b3dc5..9ec38bc015604 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/CancelAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/CancelAllocationCommand.java @@ -44,12 +44,12 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/MoveAllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/MoveAllocationCommand.java index 5a10c2796dd74..e3e152dd88889 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/MoveAllocationCommand.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/MoveAllocationCommand.java @@ -41,8 +41,8 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java index 1712d7d3ca893..807ab070b82b1 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java @@ -33,9 +33,9 @@ package org.opensearch.cluster.routing.allocation.decider; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index e216ca4511bff..4b69c05807ae4 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -52,8 +52,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.snapshots.SnapshotShardSizeInfo; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java index 8349e69791ae9..c5a54739a2494 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.RoutingAllocation; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; /** * An allocation decider that ensures we allocate the shards of a target index for resize operations next to the source primaries diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java index 4e2ab2037f548..04fa9fa45d5ea 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java @@ -9,7 +9,7 @@ package org.opensearch.cluster.service; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingStats.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingStats.java index 7340ebc97b98b..d9b292a71f625 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingStats.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingStats.java @@ -8,9 +8,9 @@ package org.opensearch.cluster.service; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java index ffc6f81490c27..790efaef95292 100644 --- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java @@ -59,7 +59,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java b/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java index f6bde29caad56..5406fa39e3059 100644 --- a/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java +++ b/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java @@ -33,10 +33,10 @@ package org.opensearch.cluster.service; import org.opensearch.common.Priority; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java index c6ac91ef71c9d..08e84355ab818 100644 --- a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java +++ b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java @@ -34,9 +34,9 @@ import com.carrotsearch.hppc.ObjectLongHashMap; import com.carrotsearch.hppc.cursors.ObjectLongCursor; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/common/Priority.java b/server/src/main/java/org/opensearch/common/Priority.java index aadaa695324b6..09a751362c945 100644 --- a/server/src/main/java/org/opensearch/common/Priority.java +++ b/server/src/main/java/org/opensearch/common/Priority.java @@ -32,8 +32,8 @@ package org.opensearch.common; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/Rounding.java b/server/src/main/java/org/opensearch/common/Rounding.java index 4fe2f8ebd2621..cae24ddee1388 100644 --- a/server/src/main/java/org/opensearch/common/Rounding.java +++ b/server/src/main/java/org/opensearch/common/Rounding.java @@ -37,9 +37,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.LocalTimeOffset.Gap; import org.opensearch.common.LocalTimeOffset.Overlap; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.time.DateUtils; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/common/Strings.java b/server/src/main/java/org/opensearch/common/Strings.java index 15a4b17252450..8e92c86836723 100644 --- a/server/src/main/java/org/opensearch/common/Strings.java +++ b/server/src/main/java/org/opensearch/common/Strings.java @@ -35,7 +35,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobStoreException.java b/server/src/main/java/org/opensearch/common/blobstore/BlobStoreException.java index 71786bb8e80dd..e60399db737b3 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/BlobStoreException.java +++ b/server/src/main/java/org/opensearch/common/blobstore/BlobStoreException.java @@ -33,7 +33,7 @@ package org.opensearch.common.blobstore; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java b/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java index 742110f898555..4cab014912970 100644 --- a/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java +++ b/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java @@ -32,10 +32,10 @@ package org.opensearch.common.breaker; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java b/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java index 7f477a4ec6a53..250ea6092c337 100644 --- a/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java +++ b/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java @@ -35,10 +35,13 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.BytesStream; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.io.IOUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/bytes/ReleasableBytesReference.java b/server/src/main/java/org/opensearch/common/bytes/ReleasableBytesReference.java index 5345c46943518..e0bc97f282e09 100644 --- a/server/src/main/java/org/opensearch/common/bytes/ReleasableBytesReference.java +++ b/server/src/main/java/org/opensearch/common/bytes/ReleasableBytesReference.java @@ -35,7 +35,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.opensearch.common.concurrent.RefCountedReleasable; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lease.Releasable; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java b/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java index f8c4dc3346b8d..2873e0d39c6ef 100644 --- a/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java +++ b/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java @@ -32,12 +32,12 @@ package org.opensearch.common.compress; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/common/compress/Compressor.java b/server/src/main/java/org/opensearch/common/compress/Compressor.java index 4f55010be2f7f..1034e3a1da5da 100644 --- a/server/src/main/java/org/opensearch/common/compress/Compressor.java +++ b/server/src/main/java/org/opensearch/common/compress/Compressor.java @@ -32,7 +32,7 @@ package org.opensearch.common.compress; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java index 4b8d0cfea57f0..62ec933fe5f37 100644 --- a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java +++ b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java @@ -33,8 +33,9 @@ package org.opensearch.common.compress; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.compress.NotXContentException; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java b/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java index 476ad818ab5ac..54cc3ad9d420e 100644 --- a/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java +++ b/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java @@ -32,7 +32,7 @@ package org.opensearch.common.compress; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.Assertions; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/common/compress/NoneCompressor.java b/server/src/main/java/org/opensearch/common/compress/NoneCompressor.java index d64645f689c67..775152b0e9388 100644 --- a/server/src/main/java/org/opensearch/common/compress/NoneCompressor.java +++ b/server/src/main/java/org/opensearch/common/compress/NoneCompressor.java @@ -8,7 +8,7 @@ package org.opensearch.common.compress; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/org/opensearch/common/compress/ZstdCompressor.java b/server/src/main/java/org/opensearch/common/compress/ZstdCompressor.java index 26b4d7e8ab569..cc3ab57e604b7 100644 --- a/server/src/main/java/org/opensearch/common/compress/ZstdCompressor.java +++ b/server/src/main/java/org/opensearch/common/compress/ZstdCompressor.java @@ -11,7 +11,7 @@ import com.github.luben.zstd.RecyclingBufferPool; import com.github.luben.zstd.ZstdInputStreamNoFinalizer; import com.github.luben.zstd.ZstdOutputStreamNoFinalizer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java index d78f435afa68e..8ce672d4fb3fc 100644 --- a/server/src/main/java/org/opensearch/common/document/DocumentField.java +++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java @@ -32,9 +32,9 @@ package org.opensearch.common.document; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -47,8 +47,8 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.parseFieldsValue; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.parseFieldsValue; /** * A single field name and values part of {@link SearchHit} and {@link GetResult}. diff --git a/server/src/main/java/org/opensearch/common/geo/GeoBoundingBox.java b/server/src/main/java/org/opensearch/common/geo/GeoBoundingBox.java index 9609c10b6614f..60ed46435874b 100644 --- a/server/src/main/java/org/opensearch/common/geo/GeoBoundingBox.java +++ b/server/src/main/java/org/opensearch/common/geo/GeoBoundingBox.java @@ -33,9 +33,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/common/geo/GeoDistance.java b/server/src/main/java/org/opensearch/common/geo/GeoDistance.java index 1c28233049376..1653f251f9201 100644 --- a/server/src/main/java/org/opensearch/common/geo/GeoDistance.java +++ b/server/src/main/java/org/opensearch/common/geo/GeoDistance.java @@ -32,9 +32,9 @@ package org.opensearch.common.geo; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.DistanceUnit; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/geo/GeoPoint.java b/server/src/main/java/org/opensearch/common/geo/GeoPoint.java index 2e03f60481509..183d3a9f77b21 100644 --- a/server/src/main/java/org/opensearch/common/geo/GeoPoint.java +++ b/server/src/main/java/org/opensearch/common/geo/GeoPoint.java @@ -40,8 +40,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchParseException; import org.opensearch.common.geo.GeoUtils.EffectivePoint; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.Geometry; diff --git a/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java b/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java index af96566ebc44a..b9d8bb22504f9 100644 --- a/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java +++ b/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java @@ -46,7 +46,7 @@ import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.common.geo.builders.ShapeBuilder.Orientation; import org.opensearch.common.geo.parsers.CoordinateNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; import org.opensearch.common.unit.DistanceUnit; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/common/geo/GeometryIO.java b/server/src/main/java/org/opensearch/common/geo/GeometryIO.java index 09bebda50047a..829ecad2aa67d 100644 --- a/server/src/main/java/org/opensearch/common/geo/GeometryIO.java +++ b/server/src/main/java/org/opensearch/common/geo/GeometryIO.java @@ -32,8 +32,8 @@ package org.opensearch.common.geo; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.geometry.Circle; import org.opensearch.geometry.Geometry; diff --git a/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java b/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java index 19b508734c245..0a5a66ef54c9c 100644 --- a/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java +++ b/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java @@ -33,9 +33,9 @@ package org.opensearch.common.geo; import org.apache.lucene.document.ShapeField.QueryRelation; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/common/geo/SpatialStrategy.java b/server/src/main/java/org/opensearch/common/geo/SpatialStrategy.java index 89bba47fc65a0..c1a13b289c6e5 100644 --- a/server/src/main/java/org/opensearch/common/geo/SpatialStrategy.java +++ b/server/src/main/java/org/opensearch/common/geo/SpatialStrategy.java @@ -31,9 +31,9 @@ package org.opensearch.common.geo; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/CircleBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/CircleBuilder.java index 4a51baca13db4..a2565f6faefd1 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/CircleBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/CircleBuilder.java @@ -38,8 +38,8 @@ import org.locationtech.spatial4j.shape.Circle; import org.locationtech.jts.geom.Coordinate; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.unit.DistanceUnit.Distance; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/EnvelopeBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/EnvelopeBuilder.java index a104ec663cedd..008e38039d64b 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/EnvelopeBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/EnvelopeBuilder.java @@ -38,8 +38,8 @@ import org.locationtech.spatial4j.shape.Rectangle; import org.locationtech.jts.geom.Coordinate; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/GeometryCollectionBuilder.java index bfb2dbaeff49e..5705f1f21701a 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/GeometryCollectionBuilder.java @@ -37,8 +37,8 @@ import org.opensearch.common.geo.XShapeCollection; import org.opensearch.common.geo.parsers.GeoWKTParser; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.Geometry; import org.opensearch.geometry.GeometryCollection; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/LineStringBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/LineStringBuilder.java index c2c091c9d8a8a..393a426c38ecd 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/LineStringBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/LineStringBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.Line; import org.locationtech.jts.geom.Coordinate; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/MultiLineStringBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/MultiLineStringBuilder.java index 1a3b8b38c6a80..5ce761f0aa64e 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/MultiLineStringBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/MultiLineStringBuilder.java @@ -35,8 +35,8 @@ import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.GeoWKTParser; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.Line; import org.opensearch.geometry.MultiLine; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/MultiPointBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/MultiPointBuilder.java index 6aefe94e1319c..b062791221fa0 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/MultiPointBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/MultiPointBuilder.java @@ -35,7 +35,7 @@ import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.XShapeCollection; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.MultiPoint; import org.locationtech.jts.geom.Coordinate; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/MultiPolygonBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/MultiPolygonBuilder.java index 3c6403818b200..907b590f268f4 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/MultiPolygonBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/MultiPolygonBuilder.java @@ -36,8 +36,8 @@ import org.opensearch.common.geo.XShapeCollection; import org.opensearch.common.geo.parsers.GeoWKTParser; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.MultiPolygon; import org.opensearch.geometry.Polygon; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/PointBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/PointBuilder.java index c26e7e592df71..bce441ae5d1d8 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/PointBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/PointBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.locationtech.jts.geom.Coordinate; import org.locationtech.spatial4j.shape.Point; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java index 99c46dc0b35cd..85cbea3142fca 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java @@ -35,8 +35,8 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.XContentBuilder; import org.locationtech.jts.geom.Coordinate; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java index 2ec70f5fc0098..a24f2541eed8a 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java @@ -42,9 +42,9 @@ import org.opensearch.common.Strings; import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.GeoWKTParser; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.locationtech.spatial4j.context.jts.JtsSpatialContext; diff --git a/server/src/main/java/org/opensearch/common/io/Streams.java b/server/src/main/java/org/opensearch/common/io/Streams.java index 6e44f18b7f4d5..b7f62fd59c3ca 100644 --- a/server/src/main/java/org/opensearch/common/io/Streams.java +++ b/server/src/main/java/org/opensearch/common/io/Streams.java @@ -32,10 +32,10 @@ package org.opensearch.common.io; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStream; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStream; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.BufferedReader; import java.io.FilterInputStream; diff --git a/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java b/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java index e6aacf7d0fcea..c2b6da3c1756a 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java @@ -35,11 +35,13 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; @@ -189,8 +191,8 @@ public BytesReference copyBytes() { } /** - * Returns the number of bytes used by the underlying {@link org.opensearch.common.util.ByteArray} - * @see org.opensearch.common.util.ByteArray#ramBytesUsed() + * Returns the number of bytes used by the underlying {@link ByteArray} + * @see ByteArray#ramBytesUsed() */ public long ramBytesUsed() { return bytes.ramBytesUsed(); diff --git a/server/src/main/java/org/opensearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/opensearch/common/io/stream/DelayableWriteable.java index 67cfe5f1025e6..200f3e9225834 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/DelayableWriteable.java +++ b/server/src/main/java/org/opensearch/common/io/stream/DelayableWriteable.java @@ -35,7 +35,12 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/io/stream/Streamables.java b/server/src/main/java/org/opensearch/common/io/stream/Streamables.java index e594247be708e..97c19f906c544 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/Streamables.java +++ b/server/src/main/java/org/opensearch/common/io/stream/Streamables.java @@ -12,7 +12,9 @@ import org.joda.time.ReadableInstant; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.time.DateUtils; -import org.opensearch.common.io.stream.Writeable.WriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable.WriteableRegistry; import org.opensearch.script.JodaCompatibleZonedDateTime; import java.time.Instant; diff --git a/server/src/main/java/org/opensearch/common/joda/Joda.java b/server/src/main/java/org/opensearch/common/joda/Joda.java index 821e0550d2af5..45700661822be 100644 --- a/server/src/main/java/org/opensearch/common/joda/Joda.java +++ b/server/src/main/java/org/opensearch/common/joda/Joda.java @@ -32,8 +32,8 @@ package org.opensearch.common.joda; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.FormatNames; diff --git a/server/src/main/java/org/opensearch/common/logging/Loggers.java b/server/src/main/java/org/opensearch/common/logging/Loggers.java index 1273e1263c5b2..7b19bd4b4be25 100644 --- a/server/src/main/java/org/opensearch/common/logging/Loggers.java +++ b/server/src/main/java/org/opensearch/common/logging/Loggers.java @@ -41,8 +41,8 @@ import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.LoggerConfig; import org.opensearch.common.settings.Setting; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index b69a429577c0f..24cc922f15a30 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -84,8 +84,8 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.iterable.Iterables; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java b/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java index f6f43cc214c5a..0ffd633e5a967 100644 --- a/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java +++ b/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.opensearch.core.Assertions; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java b/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java index 26f3bb9ccbd92..77609822d3d90 100644 --- a/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java +++ b/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java @@ -36,7 +36,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.opensearch.common.SuppressForbidden; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchLeafReader.java b/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchLeafReader.java index 10c76e43a7176..597b43cc3277f 100644 --- a/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchLeafReader.java +++ b/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchLeafReader.java @@ -34,7 +34,7 @@ import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /** * A {@link org.apache.lucene.index.FilterLeafReader} that exposes diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/CombineFunction.java b/server/src/main/java/org/opensearch/common/lucene/search/function/CombineFunction.java index 48a729d347ea1..60197ccf93af9 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/CombineFunction.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/CombineFunction.java @@ -33,9 +33,9 @@ package org.opensearch.common.lucene.search.function; import org.apache.lucene.search.Explanation; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FieldValueFactorFunction.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FieldValueFactorFunction.java index f60fd16a488a1..ee5d7424d1b71 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/FieldValueFactorFunction.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FieldValueFactorFunction.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.Explanation; import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.SortedNumericDoubleValues; diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java index 66ac66b004697..ebd155b8a5679 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java @@ -46,9 +46,9 @@ import org.apache.lucene.util.Bits; import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/metrics/OperationStats.java b/server/src/main/java/org/opensearch/common/metrics/OperationStats.java index a820f848393bb..b2135a54716a9 100644 --- a/server/src/main/java/org/opensearch/common/metrics/OperationStats.java +++ b/server/src/main/java/org/opensearch/common/metrics/OperationStats.java @@ -8,10 +8,10 @@ package org.opensearch.common.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java index 985697d46d39e..d93f8d7c98b32 100644 --- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java +++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java @@ -41,8 +41,8 @@ import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.opensearch.common.CheckedFunction; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/common/rounding/Rounding.java b/server/src/main/java/org/opensearch/common/rounding/Rounding.java index cbbbb09185ed2..9af64b8320453 100644 --- a/server/src/main/java/org/opensearch/common/rounding/Rounding.java +++ b/server/src/main/java/org/opensearch/common/rounding/Rounding.java @@ -32,9 +32,9 @@ package org.opensearch.common.rounding; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.joda.time.DateTimeField; import org.joda.time.DateTimeZone; diff --git a/server/src/main/java/org/opensearch/common/settings/BaseKeyStoreCommand.java b/server/src/main/java/org/opensearch/common/settings/BaseKeyStoreCommand.java index d35d43e53c0d6..73a2fc031fd7c 100644 --- a/server/src/main/java/org/opensearch/common/settings/BaseKeyStoreCommand.java +++ b/server/src/main/java/org/opensearch/common/settings/BaseKeyStoreCommand.java @@ -38,6 +38,7 @@ import org.opensearch.cli.KeyStoreAwareCommand; import org.opensearch.cli.Terminal; import org.opensearch.cli.UserException; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; import java.nio.file.Path; diff --git a/server/src/main/java/org/opensearch/common/settings/ChangeKeyStorePasswordCommand.java b/server/src/main/java/org/opensearch/common/settings/ChangeKeyStorePasswordCommand.java index cae8ca6002252..74e09f6f233d5 100644 --- a/server/src/main/java/org/opensearch/common/settings/ChangeKeyStorePasswordCommand.java +++ b/server/src/main/java/org/opensearch/common/settings/ChangeKeyStorePasswordCommand.java @@ -36,6 +36,7 @@ import org.opensearch.cli.ExitCodes; import org.opensearch.cli.Terminal; import org.opensearch.cli.UserException; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; /** diff --git a/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java index ffd980f8f0da6..257c4c28f0045 100644 --- a/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java @@ -45,6 +45,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; import org.opensearch.common.hash.MessageDigests; +import org.opensearch.core.common.settings.SecureString; import javax.crypto.AEADBadTagException; import javax.crypto.Cipher; diff --git a/server/src/main/java/org/opensearch/common/settings/NoClassSettingsException.java b/server/src/main/java/org/opensearch/common/settings/NoClassSettingsException.java index dc5d96536fc2c..12d7df66deddf 100644 --- a/server/src/main/java/org/opensearch/common/settings/NoClassSettingsException.java +++ b/server/src/main/java/org/opensearch/common/settings/NoClassSettingsException.java @@ -32,7 +32,7 @@ package org.opensearch.common.settings; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/settings/SecureSetting.java b/server/src/main/java/org/opensearch/common/settings/SecureSetting.java index 8e304a506399a..139c6f651a6f1 100644 --- a/server/src/main/java/org/opensearch/common/settings/SecureSetting.java +++ b/server/src/main/java/org/opensearch/common/settings/SecureSetting.java @@ -42,6 +42,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.settings.SecureString; /** * A secure setting. diff --git a/server/src/main/java/org/opensearch/common/settings/SecureSettings.java b/server/src/main/java/org/opensearch/common/settings/SecureSettings.java index e9829aac46716..2fe7d4834c92a 100644 --- a/server/src/main/java/org/opensearch/common/settings/SecureSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/SecureSettings.java @@ -32,6 +32,8 @@ package org.opensearch.common.settings; +import org.opensearch.core.common.settings.SecureString; + import java.io.Closeable; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index 13ab6b739371c..3bf2988e88e5a 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -40,9 +40,9 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Strings; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.MemorySizeValue; diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 7ea022f7f90e5..a7649a1cd22c5 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -39,8 +39,8 @@ import org.opensearch.common.Booleans; import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.unit.ByteSizeUnit; @@ -49,8 +49,9 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/common/settings/SettingsException.java b/server/src/main/java/org/opensearch/common/settings/SettingsException.java index cdf9ea11a6932..d1b924827a651 100644 --- a/server/src/main/java/org/opensearch/common/settings/SettingsException.java +++ b/server/src/main/java/org/opensearch/common/settings/SettingsException.java @@ -33,8 +33,8 @@ package org.opensearch.common.settings; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java b/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java index 4b34e614f181c..8664b14119694 100644 --- a/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java +++ b/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java @@ -10,9 +10,9 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting.ByteSizeValueParser; import org.opensearch.common.settings.Setting.DoubleParser; import org.opensearch.common.settings.Setting.FloatParser; diff --git a/server/src/main/java/org/opensearch/common/transport/BoundTransportAddress.java b/server/src/main/java/org/opensearch/common/transport/BoundTransportAddress.java index 2a131c6704724..3a9c337f2d950 100644 --- a/server/src/main/java/org/opensearch/common/transport/BoundTransportAddress.java +++ b/server/src/main/java/org/opensearch/common/transport/BoundTransportAddress.java @@ -32,9 +32,9 @@ package org.opensearch.common.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.network.InetAddresses; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/transport/TransportAddress.java b/server/src/main/java/org/opensearch/common/transport/TransportAddress.java index 086fe3ee780d8..737e8f3496143 100644 --- a/server/src/main/java/org/opensearch/common/transport/TransportAddress.java +++ b/server/src/main/java/org/opensearch/common/transport/TransportAddress.java @@ -32,9 +32,9 @@ package org.opensearch.common.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.network.NetworkAddress; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java b/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java index 0a407bb3fe4bc..b95e39feb8fac 100644 --- a/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java +++ b/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java @@ -32,9 +32,9 @@ package org.opensearch.common.unit; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java b/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java index d86ce7b7f7f87..a123c79464727 100644 --- a/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java +++ b/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.network.NetworkService; diff --git a/server/src/main/java/org/opensearch/common/unit/DistanceUnit.java b/server/src/main/java/org/opensearch/common/unit/DistanceUnit.java index 0ca9dfe5f394e..e97438f0f47c0 100644 --- a/server/src/main/java/org/opensearch/common/unit/DistanceUnit.java +++ b/server/src/main/java/org/opensearch/common/unit/DistanceUnit.java @@ -33,9 +33,9 @@ package org.opensearch.common.unit; import org.opensearch.common.geo.GeoUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/unit/Fuzziness.java b/server/src/main/java/org/opensearch/common/unit/Fuzziness.java index 894bd9738b864..2ce86cbfd13f3 100644 --- a/server/src/main/java/org/opensearch/common/unit/Fuzziness.java +++ b/server/src/main/java/org/opensearch/common/unit/Fuzziness.java @@ -33,9 +33,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/common/unit/SizeValue.java b/server/src/main/java/org/opensearch/common/unit/SizeValue.java index bebb3fdf2a895..3b73955f8d046 100644 --- a/server/src/main/java/org/opensearch/common/unit/SizeValue.java +++ b/server/src/main/java/org/opensearch/common/unit/SizeValue.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/util/AbstractArray.java b/server/src/main/java/org/opensearch/common/util/AbstractArray.java index c3560ec1e8665..4f6740c6eb89c 100644 --- a/server/src/main/java/org/opensearch/common/util/AbstractArray.java +++ b/server/src/main/java/org/opensearch/common/util/AbstractArray.java @@ -33,6 +33,7 @@ package org.opensearch.common.util; import org.apache.lucene.util.Accountable; +import org.opensearch.core.common.util.BigArray; import java.util.Collection; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 475b1d4c81ad2..45c2092601a7b 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -41,6 +41,8 @@ import org.opensearch.common.recycler.Recycler; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.common.util.BigArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.indices.breaker.CircuitBreakerService; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/common/util/BigByteArray.java b/server/src/main/java/org/opensearch/common/util/BigByteArray.java index db1d1014ffed2..5a715cf56108f 100644 --- a/server/src/main/java/org/opensearch/common/util/BigByteArray.java +++ b/server/src/main/java/org/opensearch/common/util/BigByteArray.java @@ -35,6 +35,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; +import org.opensearch.core.common.util.ByteArray; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/common/util/BytesRefHash.java b/server/src/main/java/org/opensearch/common/util/BytesRefHash.java index eb44b30a7e36c..da11d91b1f535 100644 --- a/server/src/main/java/org/opensearch/common/util/BytesRefHash.java +++ b/server/src/main/java/org/opensearch/common/util/BytesRefHash.java @@ -36,6 +36,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.common.util.ByteArray; /** * Specialized hash table implementation similar to Lucene's BytesRefHash that maps diff --git a/server/src/main/java/org/opensearch/common/util/CancellableThreads.java b/server/src/main/java/org/opensearch/common/util/CancellableThreads.java index ad7ceff90c55d..8bc3ca3affb12 100644 --- a/server/src/main/java/org/opensearch/common/util/CancellableThreads.java +++ b/server/src/main/java/org/opensearch/common/util/CancellableThreads.java @@ -35,7 +35,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.util.HashSet; diff --git a/server/src/main/java/org/opensearch/common/util/CuckooFilter.java b/server/src/main/java/org/opensearch/common/util/CuckooFilter.java index 8ef48c8d6e1a4..0c792b37ccfa9 100644 --- a/server/src/main/java/org/opensearch/common/util/CuckooFilter.java +++ b/server/src/main/java/org/opensearch/common/util/CuckooFilter.java @@ -35,9 +35,9 @@ import org.apache.lucene.store.DataOutput; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.XPackedInts; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/common/util/DoubleArray.java b/server/src/main/java/org/opensearch/common/util/DoubleArray.java index 45e57085dcca4..ae66902f6f795 100644 --- a/server/src/main/java/org/opensearch/common/util/DoubleArray.java +++ b/server/src/main/java/org/opensearch/common/util/DoubleArray.java @@ -32,6 +32,8 @@ package org.opensearch.common.util; +import org.opensearch.core.common.util.BigArray; + /** * Abstraction of an array of double values. * diff --git a/server/src/main/java/org/opensearch/common/util/FloatArray.java b/server/src/main/java/org/opensearch/common/util/FloatArray.java index b402b8e81aa43..81f6bebb7508c 100644 --- a/server/src/main/java/org/opensearch/common/util/FloatArray.java +++ b/server/src/main/java/org/opensearch/common/util/FloatArray.java @@ -32,6 +32,8 @@ package org.opensearch.common.util; +import org.opensearch.core.common.util.BigArray; + /** * Abstraction of an array of double values. * diff --git a/server/src/main/java/org/opensearch/common/util/IntArray.java b/server/src/main/java/org/opensearch/common/util/IntArray.java index 4a6e8fe5a6f95..eb43e013f3758 100644 --- a/server/src/main/java/org/opensearch/common/util/IntArray.java +++ b/server/src/main/java/org/opensearch/common/util/IntArray.java @@ -32,6 +32,8 @@ package org.opensearch.common.util; +import org.opensearch.core.common.util.BigArray; + /** * Abstraction of an array of integer values. * diff --git a/server/src/main/java/org/opensearch/common/util/LongArray.java b/server/src/main/java/org/opensearch/common/util/LongArray.java index c794af8504398..0c32effd04cdc 100644 --- a/server/src/main/java/org/opensearch/common/util/LongArray.java +++ b/server/src/main/java/org/opensearch/common/util/LongArray.java @@ -32,6 +32,8 @@ package org.opensearch.common.util; +import org.opensearch.core.common.util.BigArray; + /** * Abstraction of an array of long values. * diff --git a/server/src/main/java/org/opensearch/common/util/ObjectArray.java b/server/src/main/java/org/opensearch/common/util/ObjectArray.java index 142a722821e90..9c882742b56ab 100644 --- a/server/src/main/java/org/opensearch/common/util/ObjectArray.java +++ b/server/src/main/java/org/opensearch/common/util/ObjectArray.java @@ -32,6 +32,8 @@ package org.opensearch.common.util; +import org.opensearch.core.common.util.BigArray; + /** * Abstraction of an array of object values. * diff --git a/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java b/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java index 429dce6a69e75..f8c690471fb61 100644 --- a/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java +++ b/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java @@ -33,7 +33,7 @@ package org.opensearch.common.util; import org.apache.lucene.util.RamUsageEstimator; -import org.opensearch.common.bytes.PagedBytesReference; +import org.opensearch.core.common.bytes.PagedBytesReference; import org.opensearch.common.recycler.AbstractRecyclerC; import org.opensearch.common.recycler.Recycler; import org.opensearch.common.settings.Setting; diff --git a/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java b/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java index c8e60bddb8d0f..e99eb751babe8 100644 --- a/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java +++ b/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java @@ -34,9 +34,9 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.hash.MurmurHash3; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java index 9dd5d21a00231..1e1d70f1468dd 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java @@ -37,9 +37,9 @@ import org.opensearch.client.OriginSettingClient; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/UncategorizedExecutionException.java b/server/src/main/java/org/opensearch/common/util/concurrent/UncategorizedExecutionException.java index 5d99a346f6fb0..0d0907b272b76 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/UncategorizedExecutionException.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/UncategorizedExecutionException.java @@ -33,7 +33,7 @@ package org.opensearch.common.util.concurrent; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 0859738478fc2..7510c712e3b4b 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -8,7 +8,7 @@ package org.opensearch.common.xcontent; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.AbstractXContentParser; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/server/src/main/java/org/opensearch/common/xcontent/ObjectParserHelper.java b/server/src/main/java/org/opensearch/common/xcontent/ObjectParserHelper.java index be98d1cf73515..77c13dc5de60e 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/ObjectParserHelper.java +++ b/server/src/main/java/org/opensearch/common/xcontent/ObjectParserHelper.java @@ -34,7 +34,7 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.AbstractObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; diff --git a/server/src/main/java/org/opensearch/common/xcontent/ParseFieldRegistry.java b/server/src/main/java/org/opensearch/common/xcontent/ParseFieldRegistry.java index cb8e95c3af2e2..9770f41bce8ff 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/ParseFieldRegistry.java +++ b/server/src/main/java/org/opensearch/common/xcontent/ParseFieldRegistry.java @@ -33,7 +33,7 @@ package org.opensearch.common.xcontent; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.XContentLocation; diff --git a/server/src/main/java/org/opensearch/common/xcontent/StatusToXContentObject.java b/server/src/main/java/org/opensearch/common/xcontent/StatusToXContentObject.java index 967860d85c3fb..475f6b46555aa 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/StatusToXContentObject.java +++ b/server/src/main/java/org/opensearch/common/xcontent/StatusToXContentObject.java @@ -32,7 +32,7 @@ package org.opensearch.common.xcontent; import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; /** * Objects that can both render themselves in as json/yaml/etc and can provide a {@link RestStatus} for their response. Usually should be diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java index 482549ae194bd..cfe8008a4adca 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java @@ -34,8 +34,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.compress.Compressor; import org.opensearch.common.compress.CompressorFactory; diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java b/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java index f35bb8067a984..924db8bdea1dd 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java +++ b/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java @@ -33,7 +33,7 @@ package org.opensearch.common.xcontent; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/discovery/ClusterManagerNotDiscoveredException.java b/server/src/main/java/org/opensearch/discovery/ClusterManagerNotDiscoveredException.java index 1f2db95ed4203..7ee11df3d4e6d 100644 --- a/server/src/main/java/org/opensearch/discovery/ClusterManagerNotDiscoveredException.java +++ b/server/src/main/java/org/opensearch/discovery/ClusterManagerNotDiscoveredException.java @@ -33,8 +33,8 @@ package org.opensearch.discovery; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index 44f44fa055b2b..cf2f7b47288fd 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.service.ClusterApplier; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.common.Randomness; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java b/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java index b006119aee83b..cc19112dcfc83 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java @@ -32,9 +32,9 @@ package org.opensearch.discovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.cluster.coordination.PendingClusterStateStats; diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java index b83e9080fa452..33cdad3045780 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java @@ -9,8 +9,8 @@ package org.opensearch.discovery; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionResponse.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionResponse.java index 1fc0ee7dd325c..f56ffc84a7909 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionResponse.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionResponse.java @@ -32,8 +32,8 @@ package org.opensearch.discovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/discovery/MasterNotDiscoveredException.java b/server/src/main/java/org/opensearch/discovery/MasterNotDiscoveredException.java index 7ed21f924b27c..3915ae5ead999 100644 --- a/server/src/main/java/org/opensearch/discovery/MasterNotDiscoveredException.java +++ b/server/src/main/java/org/opensearch/discovery/MasterNotDiscoveredException.java @@ -32,7 +32,7 @@ package org.opensearch.discovery; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/discovery/PeerFinder.java b/server/src/main/java/org/opensearch/discovery/PeerFinder.java index 6bd87a4e6cdd0..96556d1cd71ed 100644 --- a/server/src/main/java/org/opensearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/opensearch/discovery/PeerFinder.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/main/java/org/opensearch/discovery/PeersRequest.java b/server/src/main/java/org/opensearch/discovery/PeersRequest.java index 83545acf80ba3..8a07571470804 100644 --- a/server/src/main/java/org/opensearch/discovery/PeersRequest.java +++ b/server/src/main/java/org/opensearch/discovery/PeersRequest.java @@ -33,8 +33,8 @@ package org.opensearch.discovery; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/env/EnvironmentSettingsResponse.java b/server/src/main/java/org/opensearch/env/EnvironmentSettingsResponse.java index 3fbe636803e66..ce3aa0556744b 100644 --- a/server/src/main/java/org/opensearch/env/EnvironmentSettingsResponse.java +++ b/server/src/main/java/org/opensearch/env/EnvironmentSettingsResponse.java @@ -8,8 +8,8 @@ package org.opensearch.env; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.transport.TransportResponse; diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java index f4a7066fa709b..f7d1f6e4343cc 100644 --- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java @@ -65,9 +65,9 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.MetadataStateFormat; import org.opensearch.gateway.PersistedClusterStateService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.FsDirectoryFactory; import org.opensearch.monitor.fs.FsInfo; diff --git a/server/src/main/java/org/opensearch/env/ShardLock.java b/server/src/main/java/org/opensearch/env/ShardLock.java index 6e4cf7ebef016..dd34eb3275f68 100644 --- a/server/src/main/java/org/opensearch/env/ShardLock.java +++ b/server/src/main/java/org/opensearch/env/ShardLock.java @@ -32,7 +32,7 @@ package org.opensearch.env; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.Closeable; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/server/src/main/java/org/opensearch/env/ShardLockObtainFailedException.java b/server/src/main/java/org/opensearch/env/ShardLockObtainFailedException.java index ea0715e063907..525d8a76c9699 100644 --- a/server/src/main/java/org/opensearch/env/ShardLockObtainFailedException.java +++ b/server/src/main/java/org/opensearch/env/ShardLockObtainFailedException.java @@ -33,8 +33,8 @@ package org.opensearch.env; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/AcknowledgedResponse.java b/server/src/main/java/org/opensearch/extensions/AcknowledgedResponse.java index be7eb9c03076e..7c7e3e78798e8 100644 --- a/server/src/main/java/org/opensearch/extensions/AcknowledgedResponse.java +++ b/server/src/main/java/org/opensearch/extensions/AcknowledgedResponse.java @@ -8,8 +8,8 @@ package org.opensearch.extensions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/extensions/AddSettingsUpdateConsumerRequest.java b/server/src/main/java/org/opensearch/extensions/AddSettingsUpdateConsumerRequest.java index 0ecf841e49b9e..c8eb65e2a94d4 100644 --- a/server/src/main/java/org/opensearch/extensions/AddSettingsUpdateConsumerRequest.java +++ b/server/src/main/java/org/opensearch/extensions/AddSettingsUpdateConsumerRequest.java @@ -8,8 +8,8 @@ package org.opensearch.extensions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.WriteableSetting; diff --git a/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java b/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java index c71eb896b9221..ac1dfe5309ffa 100644 --- a/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java +++ b/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java @@ -12,9 +12,9 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java b/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java index 56c9f0387b13e..245abc558e5fa 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java @@ -13,9 +13,9 @@ import java.util.Objects; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; /** * This class handles the dependent extensions information diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionDependencyResponse.java b/server/src/main/java/org/opensearch/extensions/ExtensionDependencyResponse.java index f486cbf1f128f..d9531c0cc2894 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionDependencyResponse.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionDependencyResponse.java @@ -13,8 +13,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; /** diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionRequest.java b/server/src/main/java/org/opensearch/extensions/ExtensionRequest.java index cd2dc99b8de93..19424e5213151 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionRequest.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionRequest.java @@ -11,8 +11,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.proto.ExtensionIdentityProto; import org.opensearch.extensions.proto.ExtensionRequestProto; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index cb22c8d864b1b..468de4238f879 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -29,12 +29,12 @@ import org.opensearch.cluster.ClusterSettingsResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Setting; -import org.opensearch.core.common.Strings; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.transport.TransportAddress; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.discovery.InitializeExtensionRequest; import org.opensearch.discovery.InitializeExtensionResponse; diff --git a/server/src/main/java/org/opensearch/extensions/OpenSearchRequest.java b/server/src/main/java/org/opensearch/extensions/OpenSearchRequest.java index 62e66f09eb856..63ddfdc91eb7a 100644 --- a/server/src/main/java/org/opensearch/extensions/OpenSearchRequest.java +++ b/server/src/main/java/org/opensearch/extensions/OpenSearchRequest.java @@ -10,8 +10,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/UpdateSettingsRequest.java b/server/src/main/java/org/opensearch/extensions/UpdateSettingsRequest.java index 6ed7b9a5a6d36..0295269b1787f 100644 --- a/server/src/main/java/org/opensearch/extensions/UpdateSettingsRequest.java +++ b/server/src/main/java/org/opensearch/extensions/UpdateSettingsRequest.java @@ -12,8 +12,8 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.WriteableSetting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java index 9bf53812c223e..bc6f6f0d688f5 100644 --- a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java +++ b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportResponseHandler; diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionActionRequest.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionActionRequest.java index cf9428cecb1b5..30a864f3f5fcf 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionActionRequest.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionActionRequest.java @@ -11,8 +11,8 @@ import com.google.protobuf.ByteString; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.proto.ExtensionTransportMessageProto.ExtensionTransportMessage; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionActionResponse.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionActionResponse.java index 68729ada48c25..6b026b51ed2aa 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionActionResponse.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionActionResponse.java @@ -9,8 +9,8 @@ package org.opensearch.extensions.action; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionActionUtil.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionActionUtil.java index 8b898b3afb1e2..93d8ff548f7fc 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionActionUtil.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionActionUtil.java @@ -9,10 +9,10 @@ package org.opensearch.extensions.action; import org.opensearch.action.ActionRequest; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.lang.reflect.Constructor; diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionHandleTransportRequest.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionHandleTransportRequest.java index d1ad4ebb7dfff..02b2a8061baa4 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionHandleTransportRequest.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionHandleTransportRequest.java @@ -9,8 +9,8 @@ package org.opensearch.extensions.action; import com.google.protobuf.ByteString; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.proto.ExtensionTransportMessageProto.ExtensionTransportMessage; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java index 3fba76b7d3c59..22502509634c6 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java @@ -15,7 +15,7 @@ import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.support.ActionFilters; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.AcknowledgedResponse; import org.opensearch.extensions.ExtensionsManager; diff --git a/server/src/main/java/org/opensearch/extensions/action/RegisterTransportActionsRequest.java b/server/src/main/java/org/opensearch/extensions/action/RegisterTransportActionsRequest.java index acfacac051803..cd6d01f0c6621 100644 --- a/server/src/main/java/org/opensearch/extensions/action/RegisterTransportActionsRequest.java +++ b/server/src/main/java/org/opensearch/extensions/action/RegisterTransportActionsRequest.java @@ -8,8 +8,8 @@ package org.opensearch.extensions.action; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.proto.RegisterTransportActionsProto.RegisterTransportActions; import org.opensearch.extensions.proto.ExtensionIdentityProto.ExtensionIdentity; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/extensions/action/RemoteExtensionActionResponse.java b/server/src/main/java/org/opensearch/extensions/action/RemoteExtensionActionResponse.java index adc269a037231..7a6b053eb7480 100644 --- a/server/src/main/java/org/opensearch/extensions/action/RemoteExtensionActionResponse.java +++ b/server/src/main/java/org/opensearch/extensions/action/RemoteExtensionActionResponse.java @@ -9,8 +9,8 @@ package org.opensearch.extensions.action; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/server/src/main/java/org/opensearch/extensions/action/TransportActionRequestFromExtension.java b/server/src/main/java/org/opensearch/extensions/action/TransportActionRequestFromExtension.java index 5338d5ae2fb15..e0ce4488fff3e 100644 --- a/server/src/main/java/org/opensearch/extensions/action/TransportActionRequestFromExtension.java +++ b/server/src/main/java/org/opensearch/extensions/action/TransportActionRequestFromExtension.java @@ -11,8 +11,8 @@ import com.google.protobuf.ByteString; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.proto.ExtensionIdentityProto.ExtensionIdentity; import org.opensearch.extensions.proto.ExtensionTransportMessageProto.ExtensionTransportMessage; diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java index 93ef9d3034062..0992345735176 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java @@ -9,9 +9,9 @@ package org.opensearch.extensions.rest; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java index 5cbc877f39eda..294f534e3f2fe 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java @@ -8,11 +8,11 @@ package org.opensearch.extensions.rest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.List; diff --git a/server/src/main/java/org/opensearch/extensions/rest/RegisterRestActionsRequest.java b/server/src/main/java/org/opensearch/extensions/rest/RegisterRestActionsRequest.java index d518f4c83fd30..3453375969b5c 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RegisterRestActionsRequest.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RegisterRestActionsRequest.java @@ -8,8 +8,8 @@ package org.opensearch.extensions.rest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.extensions.proto.ExtensionIdentityProto.ExtensionIdentity; import org.opensearch.extensions.proto.RegisterRestActionsProto.RegisterRestActions; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestExecuteOnExtensionResponse.java b/server/src/main/java/org/opensearch/extensions/rest/RestExecuteOnExtensionResponse.java index e2625105e705c..63ae6ce93af22 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestExecuteOnExtensionResponse.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestExecuteOnExtensionResponse.java @@ -8,10 +8,10 @@ package org.opensearch.extensions.rest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 878673b77a4a9..2a3ad63379556 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -14,6 +14,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.extensions.ExtensionDependency; @@ -24,7 +25,6 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; import org.opensearch.transport.ConnectTransportException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index 3dd6056bb36cf..e20a3c6b2814c 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -12,8 +12,8 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.ExtensionsManager; @@ -22,7 +22,7 @@ import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestRequest.Method; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportResponseHandler; diff --git a/server/src/main/java/org/opensearch/extensions/settings/RegisterCustomSettingsRequest.java b/server/src/main/java/org/opensearch/extensions/settings/RegisterCustomSettingsRequest.java index 1f7e23544ebdf..e3386c2838755 100644 --- a/server/src/main/java/org/opensearch/extensions/settings/RegisterCustomSettingsRequest.java +++ b/server/src/main/java/org/opensearch/extensions/settings/RegisterCustomSettingsRequest.java @@ -8,8 +8,8 @@ package org.opensearch.extensions.settings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.WriteableSetting; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java index e4df2e604c320..937432096817e 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java @@ -45,7 +45,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.ReceiveTimeoutTransportException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java b/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java index bf2d37bdcb3f1..dba081f1b50c2 100644 --- a/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java +++ b/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java @@ -45,7 +45,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/gateway/Gateway.java b/server/src/main/java/org/opensearch/gateway/Gateway.java index 27e028ced0c54..15b226e2d30e2 100644 --- a/server/src/main/java/org/opensearch/gateway/Gateway.java +++ b/server/src/main/java/org/opensearch/gateway/Gateway.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Arrays; import java.util.function.Function; diff --git a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java index 1a4681766e489..5a20112b19219 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java @@ -52,7 +52,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.set.Sets; import org.opensearch.common.lease.Releasables; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/gateway/GatewayException.java b/server/src/main/java/org/opensearch/gateway/GatewayException.java index 59313a3fe84a4..84593d2a1ae9d 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayException.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayException.java @@ -33,7 +33,7 @@ package org.opensearch.gateway; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/gateway/GatewayService.java b/server/src/main/java/org/opensearch/gateway/GatewayService.java index d04f3ee15d888..cf105380e98ad 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayService.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayService.java @@ -54,7 +54,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.discovery.Discovery; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/server/src/main/java/org/opensearch/gateway/IncrementalClusterStateWriter.java b/server/src/main/java/org/opensearch/gateway/IncrementalClusterStateWriter.java index 3fdde3a3bad09..ec599df7f7426 100644 --- a/server/src/main/java/org/opensearch/gateway/IncrementalClusterStateWriter.java +++ b/server/src/main/java/org/opensearch/gateway/IncrementalClusterStateWriter.java @@ -42,7 +42,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.ArrayList; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/opensearch/gateway/LocalAllocateDangledIndices.java index e2666491630b7..5ee369d6b9402 100644 --- a/server/src/main/java/org/opensearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/opensearch/gateway/LocalAllocateDangledIndices.java @@ -50,8 +50,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/gateway/MetaStateService.java b/server/src/main/java/org/opensearch/gateway/MetaStateService.java index 7870b3e205a3a..ce912142e35b0 100644 --- a/server/src/main/java/org/opensearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/opensearch/gateway/MetaStateService.java @@ -43,7 +43,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java index 2515cd631481e..8940b0ed25ed4 100644 --- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java @@ -66,7 +66,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.RecyclingBytesStreamOutput; import org.opensearch.common.io.Streams; import org.opensearch.common.logging.Loggers; @@ -75,7 +75,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; @@ -88,7 +88,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeMetadata; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.Closeable; import java.io.IOError; diff --git a/server/src/main/java/org/opensearch/gateway/PriorityComparator.java b/server/src/main/java/org/opensearch/gateway/PriorityComparator.java index 7b3179a509057..7b8d2ca8e2787 100644 --- a/server/src/main/java/org/opensearch/gateway/PriorityComparator.java +++ b/server/src/main/java/org/opensearch/gateway/PriorityComparator.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java index ba1490a7929bd..3be8ac9784960 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java @@ -47,8 +47,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java index b529557aa9815..e2a3f08bb02c6 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java @@ -49,14 +49,14 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.shard.ShardStateMetadata; import org.opensearch.index.store.Store; diff --git a/server/src/main/java/org/opensearch/http/BindHttpException.java b/server/src/main/java/org/opensearch/http/BindHttpException.java index 0f0e7198db548..f7a184e391fa9 100644 --- a/server/src/main/java/org/opensearch/http/BindHttpException.java +++ b/server/src/main/java/org/opensearch/http/BindHttpException.java @@ -32,7 +32,7 @@ package org.opensearch.http; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/http/CorsHandler.java b/server/src/main/java/org/opensearch/http/CorsHandler.java index 4f69a938ea84c..4049de01175a6 100644 --- a/server/src/main/java/org/opensearch/http/CorsHandler.java +++ b/server/src/main/java/org/opensearch/http/CorsHandler.java @@ -47,12 +47,12 @@ package org.opensearch.http; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.core.common.Strings; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import java.time.ZoneOffset; diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index bee7ae9d31d27..75a2a68dd78fe 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.network.CloseableChannel; @@ -47,7 +47,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.ArrayList; import java.util.List; diff --git a/server/src/main/java/org/opensearch/http/HttpException.java b/server/src/main/java/org/opensearch/http/HttpException.java index c5eb734bbe5fc..ec78702c77bfe 100644 --- a/server/src/main/java/org/opensearch/http/HttpException.java +++ b/server/src/main/java/org/opensearch/http/HttpException.java @@ -33,7 +33,7 @@ package org.opensearch.http; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/http/HttpInfo.java b/server/src/main/java/org/opensearch/http/HttpInfo.java index e55eeade72211..a5e981e98e3de 100644 --- a/server/src/main/java/org/opensearch/http/HttpInfo.java +++ b/server/src/main/java/org/opensearch/http/HttpInfo.java @@ -32,8 +32,8 @@ package org.opensearch.http; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.transport.BoundTransportAddress; diff --git a/server/src/main/java/org/opensearch/http/HttpPipelinedRequest.java b/server/src/main/java/org/opensearch/http/HttpPipelinedRequest.java index 852c35e2417d4..e88adaf5387e6 100644 --- a/server/src/main/java/org/opensearch/http/HttpPipelinedRequest.java +++ b/server/src/main/java/org/opensearch/http/HttpPipelinedRequest.java @@ -32,9 +32,9 @@ package org.opensearch.http; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.List; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/http/HttpRequest.java b/server/src/main/java/org/opensearch/http/HttpRequest.java index c78db0f40c9f0..639bf40cad375 100644 --- a/server/src/main/java/org/opensearch/http/HttpRequest.java +++ b/server/src/main/java/org/opensearch/http/HttpRequest.java @@ -33,9 +33,9 @@ package org.opensearch.http; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.Collections; import java.util.List; diff --git a/server/src/main/java/org/opensearch/http/HttpStats.java b/server/src/main/java/org/opensearch/http/HttpStats.java index 6dd2cf792db96..078b84b7bc563 100644 --- a/server/src/main/java/org/opensearch/http/HttpStats.java +++ b/server/src/main/java/org/opensearch/http/HttpStats.java @@ -32,9 +32,9 @@ package org.opensearch.http; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/AbstractIndexComponent.java b/server/src/main/java/org/opensearch/index/AbstractIndexComponent.java index 88809156174c0..7b3705f921954 100644 --- a/server/src/main/java/org/opensearch/index/AbstractIndexComponent.java +++ b/server/src/main/java/org/opensearch/index/AbstractIndexComponent.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.logging.Loggers; +import org.opensearch.core.index.Index; /** * A base class for new index components diff --git a/server/src/main/java/org/opensearch/index/CompositeIndexEventListener.java b/server/src/main/java/org/opensearch/index/CompositeIndexEventListener.java index 3b5ff7798e0de..2090d9a1ce8f0 100644 --- a/server/src/main/java/org/opensearch/index/CompositeIndexEventListener.java +++ b/server/src/main/java/org/opensearch/index/CompositeIndexEventListener.java @@ -38,10 +38,11 @@ import org.opensearch.common.Nullable; import org.opensearch.common.logging.Loggers; import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/IndexComponent.java b/server/src/main/java/org/opensearch/index/IndexComponent.java index 2a4555b66df13..fe57abe2c907e 100644 --- a/server/src/main/java/org/opensearch/index/IndexComponent.java +++ b/server/src/main/java/org/opensearch/index/IndexComponent.java @@ -32,6 +32,8 @@ package org.opensearch.index; +import org.opensearch.core.index.Index; + /** * Actions that can be executed on an Index Component * diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index bdb043b7b9aa1..e4e3a79c8e60c 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -48,12 +48,13 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.common.SetOnce; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; diff --git a/server/src/main/java/org/opensearch/index/IndexNotFoundException.java b/server/src/main/java/org/opensearch/index/IndexNotFoundException.java index 120e7f955cd9c..f1459fcffa395 100644 --- a/server/src/main/java/org/opensearch/index/IndexNotFoundException.java +++ b/server/src/main/java/org/opensearch/index/IndexNotFoundException.java @@ -32,7 +32,8 @@ package org.opensearch.index; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e6930b41088e2..1ecc98b7f69f2 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -49,7 +49,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedFunction; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; @@ -83,7 +83,7 @@ import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.SearchOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.index.shard.ShardPath; diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index dae2d2369fc4c..30c9efde19cdc 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -47,6 +47,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; +import org.opensearch.core.index.Index; import org.opensearch.index.translog.Translog; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.ingest.IngestService; diff --git a/server/src/main/java/org/opensearch/index/IndexingPressureService.java b/server/src/main/java/org/opensearch/index/IndexingPressureService.java index c28c18eb5517f..35e022df22694 100644 --- a/server/src/main/java/org/opensearch/index/IndexingPressureService.java +++ b/server/src/main/java/org/opensearch/index/IndexingPressureService.java @@ -9,7 +9,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.stats.IndexingPressureStats; import org.opensearch.index.stats.ShardIndexingPressureStats; diff --git a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java index d6ec06d5a274c..87605969feb6a 100644 --- a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java +++ b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java @@ -43,10 +43,11 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.Engine; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.shard.IndexingOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/server/src/main/java/org/opensearch/index/IndicesModuleRequest.java b/server/src/main/java/org/opensearch/index/IndicesModuleRequest.java index 0e0fe87df76cd..1335eb529d2f0 100644 --- a/server/src/main/java/org/opensearch/index/IndicesModuleRequest.java +++ b/server/src/main/java/org/opensearch/index/IndicesModuleRequest.java @@ -8,9 +8,10 @@ package org.opensearch.index; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/IndicesModuleResponse.java b/server/src/main/java/org/opensearch/index/IndicesModuleResponse.java index 7b41f629e48ed..67f2c686dbf8b 100644 --- a/server/src/main/java/org/opensearch/index/IndicesModuleResponse.java +++ b/server/src/main/java/org/opensearch/index/IndicesModuleResponse.java @@ -8,8 +8,8 @@ package org.opensearch.index; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java index e890473ce38f3..efbd5efbb07bc 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java @@ -8,12 +8,12 @@ package org.opensearch.index; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Set; diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java index 2740baa8ad166..7e54d5179ea35 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java @@ -22,7 +22,7 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java index eca57195db81c..b0e6e5076d03c 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java @@ -9,9 +9,9 @@ package org.opensearch.index; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationStats.java index d5ff11c692194..0f32c73f80336 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationStats.java @@ -8,12 +8,12 @@ package org.opensearch.index; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java b/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java index d7176127615d5..c9ddf494ae230 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java @@ -10,7 +10,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java index 53e0f7cea79b2..8a9f3eea6030b 100644 --- a/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java +++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java @@ -18,7 +18,7 @@ import org.opensearch.index.ShardIndexingPressureTracker.PerformanceTracker; import org.opensearch.index.ShardIndexingPressureTracker.RejectionTracker; import org.opensearch.index.ShardIndexingPressureTracker.StatsTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.stats.ShardIndexingPressureStats; import org.opensearch.index.stats.IndexingPressurePerShardStats; diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java index d64740488ce31..9f8ac7ea76cfd 100644 --- a/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java +++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java @@ -18,7 +18,7 @@ import org.opensearch.index.ShardIndexingPressureTracker.PerformanceTracker; import org.opensearch.index.ShardIndexingPressureTracker.RejectionTracker; import org.opensearch.index.ShardIndexingPressureTracker.StatsTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java index f634365676b18..b41dd1359394b 100644 --- a/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java +++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java @@ -9,7 +9,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Collections; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java index 937424598bc24..7d67b47141ef5 100644 --- a/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java +++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java @@ -5,7 +5,7 @@ package org.opensearch.index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; diff --git a/server/src/main/java/org/opensearch/index/VersionType.java b/server/src/main/java/org/opensearch/index/VersionType.java index 0eb6dfa8fc53a..111aa68152d4d 100644 --- a/server/src/main/java/org/opensearch/index/VersionType.java +++ b/server/src/main/java/org/opensearch/index/VersionType.java @@ -31,9 +31,9 @@ package org.opensearch.index; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.uid.Versions; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java b/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java index 278b44aaeff85..0295815e1c048 100644 --- a/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java +++ b/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java @@ -32,9 +32,9 @@ package org.opensearch.index.analysis; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/opensearch/index/cache/bitset/BitsetFilterCache.java index cd3691ba66a17..f4436fedb25f0 100644 --- a/server/src/main/java/org/opensearch/index/cache/bitset/BitsetFilterCache.java +++ b/server/src/main/java/org/opensearch/index/cache/bitset/BitsetFilterCache.java @@ -65,7 +65,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardUtils; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/cache/bitset/ShardBitsetFilterCache.java b/server/src/main/java/org/opensearch/index/cache/bitset/ShardBitsetFilterCache.java index 025bfe6bab666..3e7720dabad40 100644 --- a/server/src/main/java/org/opensearch/index/cache/bitset/ShardBitsetFilterCache.java +++ b/server/src/main/java/org/opensearch/index/cache/bitset/ShardBitsetFilterCache.java @@ -35,7 +35,7 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.AbstractIndexShardComponent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /** * Bitset Filter Cache for shards diff --git a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java index eed0f45e331f8..a4f2628b5e5a3 100644 --- a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java +++ b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java @@ -33,9 +33,9 @@ package org.opensearch.index.cache.query; import org.apache.lucene.search.DocIdSet; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java index 553e970bb84e3..ab8b4706e4ebe 100644 --- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java +++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.cache.request; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java index e0ab80777c82d..c3c552b5f732d 100644 --- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java +++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java @@ -33,7 +33,7 @@ package org.opensearch.index.cache.request; import org.apache.lucene.util.Accountable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.metrics.CounterMetric; /** diff --git a/server/src/main/java/org/opensearch/index/engine/CommitStats.java b/server/src/main/java/org/opensearch/index/engine/CommitStats.java index c20e5b612085b..2c78f675c5154 100644 --- a/server/src/main/java/org/opensearch/index/engine/CommitStats.java +++ b/server/src/main/java/org/opensearch/index/engine/CommitStats.java @@ -33,9 +33,9 @@ import org.apache.lucene.index.SegmentInfos; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/engine/DocumentMissingException.java b/server/src/main/java/org/opensearch/index/engine/DocumentMissingException.java index 7cb35b8279153..350db5c9bc795 100644 --- a/server/src/main/java/org/opensearch/index/engine/DocumentMissingException.java +++ b/server/src/main/java/org/opensearch/index/engine/DocumentMissingException.java @@ -31,9 +31,9 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/DocumentSourceMissingException.java b/server/src/main/java/org/opensearch/index/engine/DocumentSourceMissingException.java index b3826d9933776..0fafbb413bbeb 100644 --- a/server/src/main/java/org/opensearch/index/engine/DocumentSourceMissingException.java +++ b/server/src/main/java/org/opensearch/index/engine/DocumentSourceMissingException.java @@ -31,9 +31,9 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index 9bb99d0ef0d99..326e6aef45b08 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -59,7 +59,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.Lucene; @@ -84,7 +84,7 @@ import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.DocsStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogManager; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index f4aa5e248ac31..71bd64426161e 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -50,7 +50,7 @@ import org.opensearch.index.codec.CodecService; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.seqno.RetentionLeases; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TranslogConfig; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index 76b13ee244a2c..744df13265250 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -25,7 +25,7 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.RetentionLeases; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineCreationFailureException.java b/server/src/main/java/org/opensearch/index/engine/EngineCreationFailureException.java index b7ea337f271d1..e01b2c3e21e7e 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineCreationFailureException.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineCreationFailureException.java @@ -32,8 +32,8 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineException.java b/server/src/main/java/org/opensearch/index/engine/EngineException.java index 309f287833bb9..484bc6c7eea6a 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineException.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineException.java @@ -33,8 +33,8 @@ package org.opensearch.index.engine; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/FlushFailedEngineException.java b/server/src/main/java/org/opensearch/index/engine/FlushFailedEngineException.java index 35d8db66be9ff..9f3bad250a984 100644 --- a/server/src/main/java/org/opensearch/index/engine/FlushFailedEngineException.java +++ b/server/src/main/java/org/opensearch/index/engine/FlushFailedEngineException.java @@ -32,8 +32,8 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index c57ad88487019..77e2f5cbef7f9 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -102,7 +102,7 @@ import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.OpenSearchMergePolicy; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogCorruptedException; import org.opensearch.index.translog.TranslogDeletionPolicy; diff --git a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java index d6995eab90fdb..abde2aff6e9e6 100644 --- a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java @@ -47,7 +47,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.util.ArrayUtil; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java index 83794578ea2bf..ce0f1e85e294d 100644 --- a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java +++ b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java @@ -49,7 +49,7 @@ import org.opensearch.index.MergeSchedulerConfig; import org.opensearch.index.merge.MergeStats; import org.opensearch.index.merge.OnGoingMerge; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/index/engine/RecoveryEngineException.java b/server/src/main/java/org/opensearch/index/engine/RecoveryEngineException.java index 5894ed051eeed..9cc0eb59e95df 100644 --- a/server/src/main/java/org/opensearch/index/engine/RecoveryEngineException.java +++ b/server/src/main/java/org/opensearch/index/engine/RecoveryEngineException.java @@ -32,9 +32,9 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/RefreshFailedEngineException.java b/server/src/main/java/org/opensearch/index/engine/RefreshFailedEngineException.java index 2341466072f73..35872bb0a1fba 100644 --- a/server/src/main/java/org/opensearch/index/engine/RefreshFailedEngineException.java +++ b/server/src/main/java/org/opensearch/index/engine/RefreshFailedEngineException.java @@ -32,8 +32,8 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/Segment.java b/server/src/main/java/org/opensearch/index/engine/Segment.java index a80256cb685c1..035d5bb293303 100644 --- a/server/src/main/java/org/opensearch/index/engine/Segment.java +++ b/server/src/main/java/org/opensearch/index/engine/Segment.java @@ -40,9 +40,9 @@ import org.apache.lucene.search.SortedNumericSelector; import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.unit.ByteSizeValue; diff --git a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java index aeb67f1ee41a2..dbadd50ebd1a1 100644 --- a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java +++ b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java @@ -33,9 +33,9 @@ package org.opensearch.index.engine; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/engine/SnapshotFailedEngineException.java b/server/src/main/java/org/opensearch/index/engine/SnapshotFailedEngineException.java index 7340c21dfb321..ef5829a481366 100644 --- a/server/src/main/java/org/opensearch/index/engine/SnapshotFailedEngineException.java +++ b/server/src/main/java/org/opensearch/index/engine/SnapshotFailedEngineException.java @@ -32,7 +32,7 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/VersionConflictEngineException.java b/server/src/main/java/org/opensearch/index/engine/VersionConflictEngineException.java index 8b78b41fc3a70..75cb19a5e7443 100644 --- a/server/src/main/java/org/opensearch/index/engine/VersionConflictEngineException.java +++ b/server/src/main/java/org/opensearch/index/engine/VersionConflictEngineException.java @@ -31,10 +31,10 @@ package org.opensearch.index.engine; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java index bcb35865eafdd..896039313ea1e 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java @@ -34,9 +34,9 @@ import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataCache.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataCache.java index ef816099c7c58..72df2453fc7fd 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataCache.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataCache.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.Accountable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /** * A simple field data cache abstraction on the *index* level. diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java index 46855b9a46833..0b370893cd90d 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java @@ -40,7 +40,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.search.lookup.SearchLookup; diff --git a/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java index f816ff7a436f2..f03896c4b670a 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java @@ -39,7 +39,7 @@ import org.opensearch.common.regex.Regex; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Map; import java.util.concurrent.ConcurrentMap; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java index 4d2cb7e84df6c..b0cb473f6f08d 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.SortedBinaryDocValues; diff --git a/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java index d1c0e48570576..91ca07d753cc6 100644 --- a/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java @@ -34,8 +34,8 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.IgnoredFieldMapper; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/opensearch/index/flush/FlushStats.java b/server/src/main/java/org/opensearch/index/flush/FlushStats.java index 0af9e7e9288bb..0c05ae162782a 100644 --- a/server/src/main/java/org/opensearch/index/flush/FlushStats.java +++ b/server/src/main/java/org/opensearch/index/flush/FlushStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.flush; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/get/GetResult.java b/server/src/main/java/org/opensearch/index/get/GetResult.java index a88294943c087..65788b2297da4 100644 --- a/server/src/main/java/org/opensearch/index/get/GetResult.java +++ b/server/src/main/java/org/opensearch/index/get/GetResult.java @@ -35,12 +35,12 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; @@ -59,7 +59,7 @@ import java.util.Objects; import static java.util.Collections.emptyMap; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; diff --git a/server/src/main/java/org/opensearch/index/get/GetStats.java b/server/src/main/java/org/opensearch/index/get/GetStats.java index 3eb06b2d544e8..5837ec2bedc2e 100644 --- a/server/src/main/java/org/opensearch/index/get/GetStats.java +++ b/server/src/main/java/org/opensearch/index/get/GetStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.get; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java index 08e2b32bded0e..a815074119fb1 100644 --- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java @@ -43,7 +43,7 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.common.lucene.uid.Versions; diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index 64f9a7e6f09cd..a55e55111ffa2 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -38,8 +38,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/opensearch/index/mapper/BinaryRangeUtil.java index f24d3a5853a88..05c5e6f1424b3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryRangeUtil.java @@ -36,7 +36,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java index f271e55b9d0f6..83f0fada28467 100644 --- a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java @@ -44,7 +44,7 @@ import org.apache.lucene.search.suggest.document.SuggestField; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.util.set.Sets; diff --git a/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java index f9afdb69fe919..0a461fff38fac 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java @@ -36,7 +36,7 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; import org.opensearch.search.lookup.SearchLookup; diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java index d3584261b36d6..2461a72a2d041 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java @@ -40,10 +40,10 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchGenerationException; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperException.java b/server/src/main/java/org/opensearch/index/mapper/MapperException.java index 51f4614b883a3..75ef3c9a56be5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperException.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperException.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperParsingException.java b/server/src/main/java/org/opensearch/index/mapper/MapperParsingException.java index d293b69838b6b..7dff82e2e3167 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperParsingException.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperParsingException.java @@ -32,8 +32,8 @@ package org.opensearch.index.mapper; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java index c17444740ca30..46f0352976408 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.document.Field; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.mapper.ParseContext.Document; import org.opensearch.index.mapper.MapperService.MergeReason; diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java index 4905d45adf189..95e71f168c2ca 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java @@ -39,7 +39,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.CollectionUtils; diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java index 8afc0e25b079d..085e258c5501e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java @@ -35,8 +35,8 @@ import java.util.Objects; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; /** diff --git a/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java b/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java index 6c2f10626541f..9127641128dad 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java +++ b/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java @@ -31,8 +31,8 @@ package org.opensearch.index.mapper; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStats.java b/server/src/main/java/org/opensearch/index/merge/MergeStats.java index d472f9a77890b..fee0ed904b68a 100644 --- a/server/src/main/java/org/opensearch/index/merge/MergeStats.java +++ b/server/src/main/java/org/opensearch/index/merge/MergeStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.merge; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java index c7072625da82b..042890c238410 100644 --- a/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java @@ -40,15 +40,15 @@ import org.opensearch.action.get.GetResponse; import org.opensearch.client.Client; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; import org.opensearch.common.geo.GeoJson; import org.opensearch.common.geo.GeometryIO; import org.opensearch.common.geo.GeometryParser; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.geo.builders.ShapeBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java index 34cefc135bc4f..c45637f986d35 100644 --- a/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java @@ -36,10 +36,10 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.xcontent.SuggestingErrorOnUnknown; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java index 1a4a9ed185def..9f62ccfe9f31f 100644 --- a/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java @@ -34,8 +34,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java index f9315aa7235b3..21735da57daf1 100644 --- a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java @@ -38,9 +38,9 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java index cd1b00c302c91..26124b422f26f 100644 --- a/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.queries.function.FunctionScoreQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java index a22dd513c41b1..e127e5a8dbc4a 100644 --- a/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java @@ -41,9 +41,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRefBuilder; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java index c9d1e10faf738..6a29ad8a0a401 100644 --- a/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java index 3d3bddca455c0..e4fda385ead97 100644 --- a/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java index 939939f88a46d..222b2cd4b1cd0 100644 --- a/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java @@ -34,11 +34,11 @@ import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java index 1d40793158a2c..38960792b1dea 100644 --- a/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java @@ -39,9 +39,9 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java index 2ff41b5f66152..1a9623f0fd7dd 100644 --- a/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java index aaedc0b92eb47..49d4c4f87d691 100644 --- a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java index 8442da6b09eb6..5cf3bf2de87c7 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -37,14 +37,14 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Numbers; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.geo.SpatialStrategy; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geometry.Rectangle; diff --git a/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java index b4d5a8b9ec2c9..f43a088434fc0 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java @@ -35,14 +35,14 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoDistance; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.geo.SpatialStrategy; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/GeoExecType.java b/server/src/main/java/org/opensearch/index/query/GeoExecType.java index 8b58b50352384..39248d4e8edb6 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoExecType.java +++ b/server/src/main/java/org/opensearch/index/query/GeoExecType.java @@ -33,9 +33,9 @@ package org.opensearch.index.query; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java index c6348d3c2e7be..a230f09ecd374 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java @@ -39,11 +39,11 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java index 78a6246440083..92768ff5f4c43 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java @@ -35,13 +35,13 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.geo.SpatialStrategy; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geometry.Geometry; diff --git a/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java b/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java index eee6782a1f85e..aad3e50a0acd5 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java +++ b/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java @@ -32,9 +32,9 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.CollectionUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java index e6600b65c1803..245c7f2240ff6 100644 --- a/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java @@ -34,9 +34,9 @@ import org.apache.lucene.search.Query; import org.opensearch.Version; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.common.Strings; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/opensearch/index/query/InnerHitBuilder.java index ffeaac88bfa56..279b9bf07171b 100644 --- a/server/src/main/java/org/opensearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/InnerHitBuilder.java @@ -32,11 +32,11 @@ package org.opensearch.index.query; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/index/query/IntervalMode.java b/server/src/main/java/org/opensearch/index/query/IntervalMode.java index 9e4d07aa9c943..454d867b41da2 100644 --- a/server/src/main/java/org/opensearch/index/query/IntervalMode.java +++ b/server/src/main/java/org/opensearch/index/query/IntervalMode.java @@ -9,9 +9,9 @@ package org.opensearch.index.query; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java index 101ad39d478bf..421f174591954 100644 --- a/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.queries.intervals.IntervalQuery; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java index 7f812f436b70f..b696539bbd366 100644 --- a/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java +++ b/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java @@ -42,11 +42,11 @@ import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java index a8227d30f8ed4..fbf2b209dc6c7 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.index.query; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java index 651bcee042fb6..8f5b537ac8e08 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java @@ -34,9 +34,9 @@ import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.LoggingDeprecationHandler; diff --git a/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java index b8d55245221ef..98a6a32de4847 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.index.query; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java index 80b6d64d82cb6..d61a5957627ea 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.search.MatchQuery; diff --git a/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java index c584c8af82e60..43e740a1d2867 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java @@ -34,9 +34,9 @@ import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java index eda55473baf6b..59aee000b7792 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.LoggingDeprecationHandler; diff --git a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java index ab711d3a0ea00..36b74c95c3200 100644 --- a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java @@ -49,12 +49,12 @@ import org.opensearch.client.Client; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.search.MoreLikeThisQuery; import org.opensearch.common.lucene.search.XMoreLikeThis; import org.opensearch.common.lucene.uid.Versions; diff --git a/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java index ee9332fecf93c..38e341d2be536 100644 --- a/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java @@ -36,10 +36,10 @@ import org.apache.lucene.search.Query; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java index 5475eca588fbe..547974b2fd5fe 100644 --- a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java @@ -50,9 +50,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.search.MaxScoreCollector; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; diff --git a/server/src/main/java/org/opensearch/index/query/Operator.java b/server/src/main/java/org/opensearch/index/query/Operator.java index 01fef29413484..31caca0f60caa 100644 --- a/server/src/main/java/org/opensearch/index/query/Operator.java +++ b/server/src/main/java/org/opensearch/index/query/Operator.java @@ -33,9 +33,9 @@ import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.BooleanClause; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.CollectionUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java index 546eb8a50f138..1956b50a39513 100644 --- a/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java @@ -37,9 +37,9 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryBuilder.java index 062f7ffa2602a..a40ccf427794a 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/QueryBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.index.query; import org.apache.lucene.search.Query; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContentObject; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilders.java b/server/src/main/java/org/opensearch/index/query/QueryBuilders.java index 0a5e846bd4e94..be384d8bf1a20 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryBuilders.java +++ b/server/src/main/java/org/opensearch/index/query/QueryBuilders.java @@ -34,7 +34,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.geo.builders.ShapeBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java index 2aa97ef2a6a64..ab5406d5531bd 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.client.Client; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index 8d8c64024eb42..a1fc4327997e8 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -42,15 +42,15 @@ import org.opensearch.action.ActionListener; import org.opensearch.client.Client; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.IndexSortConfig; import org.opensearch.index.analysis.IndexAnalyzers; diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardException.java b/server/src/main/java/org/opensearch/index/query/QueryShardException.java index 26865db0ab042..68a2501c501f3 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardException.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardException.java @@ -33,10 +33,10 @@ package org.opensearch.index.query; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java index 203ac2495f726..ebe314cebf1a0 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java @@ -37,9 +37,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.automaton.Operations; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; diff --git a/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java index 8bd668b816ae9..0f43a192cdd60 100644 --- a/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java @@ -36,10 +36,10 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.ShapeRelation; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateMathParser; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java index 8a485284128c4..0cce2401df065 100644 --- a/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java @@ -39,9 +39,9 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/index/query/Rewriteable.java b/server/src/main/java/org/opensearch/index/query/Rewriteable.java index dd148bf98ea48..ea884f720f4fc 100644 --- a/server/src/main/java/org/opensearch/index/query/Rewriteable.java +++ b/server/src/main/java/org/opensearch/index/query/Rewriteable.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; import org.opensearch.action.ActionListener; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java index 68ce127d0883d..916df7debc1bc 100644 --- a/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java @@ -45,9 +45,9 @@ import org.apache.lucene.search.Weight; import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.Functions; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java b/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java index 0d6993a42d7d7..cf9f86b6f6341 100644 --- a/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java index ef55d7e945f9c..ed4f5c6848b06 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java index 821f32363c1fd..7427b13463284 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java index c3b41d9dd7b69..9e458c994bf9e 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java @@ -40,9 +40,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopTermsRewrite; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java index 46cddf8adac4e..c86831fad0370 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java @@ -35,10 +35,10 @@ import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java index 08dc289ecbdbf..98e7f287749f5 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java index b5e9458f784f2..2f63e6d7403f7 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/SpanQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanQueryBuilder.java index 8989f05aebde1..d785cb253ece8 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanQueryBuilder.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; /** diff --git a/server/src/main/java/org/opensearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanTermQueryBuilder.java index 8b459b1d81d27..d97fbaf38fdae 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanTermQueryBuilder.java @@ -37,8 +37,8 @@ import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java index 4672c02f9b4dc..5d02cc0026dfd 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.queries.spans.SpanWithinQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/TermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermQueryBuilder.java index 7dd235781532f..21bc60646d535 100644 --- a/server/src/main/java/org/opensearch/index/query/TermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermQueryBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java index 78c4f30f9479d..c677bab0ea7e0 100644 --- a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java @@ -40,12 +40,12 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.get.GetRequest; import org.opensearch.client.Client; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java index 664d6d1dc164e..940356563ab4d 100644 --- a/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java @@ -43,9 +43,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java index b61e527b3524c..69b0190bfb700 100644 --- a/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java @@ -37,9 +37,9 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java index 30d4ce575a499..aab9adecb9a62 100644 --- a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java @@ -35,10 +35,10 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java index cab2d69ff0f9f..40b15eace2bad 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java @@ -36,13 +36,13 @@ import org.apache.lucene.search.Explanation; import org.opensearch.OpenSearchParseException; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoDistance; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.LeafScoreFunction; import org.opensearch.common.lucene.search.function.ScoreFunction; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionParser.java b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionParser.java index ea27625844337..4bbb9e32730c4 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionParser.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionParser.java @@ -33,8 +33,8 @@ package org.opensearch.index.query.functionscore; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java index c541ab84c8141..b526dab025e55 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java @@ -34,8 +34,8 @@ import org.apache.lucene.search.Explanation; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.Functions; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java index 1f8f98959c270..de22ae14528c2 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; import org.opensearch.common.lucene.search.function.ScoreFunction; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java index f5c17017e47c4..1d15172852295 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java @@ -36,10 +36,10 @@ import org.apache.lucene.search.Query; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.common.lucene.search.function.ScoreFunction; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/GaussDecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/GaussDecayFunctionBuilder.java index 977d1b774dcc9..3ef28493ed146 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/GaussDecayFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/GaussDecayFunctionBuilder.java @@ -35,8 +35,8 @@ import org.apache.lucene.search.Explanation; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.Functions; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/LinearDecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/LinearDecayFunctionBuilder.java index 0b4e84393af1c..0247af6acbd74 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/LinearDecayFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/LinearDecayFunctionBuilder.java @@ -34,8 +34,8 @@ import org.apache.lucene.search.Explanation; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.Functions; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/RandomScoreFunctionBuilder.java index 4f186fb5551aa..fa960099c7e3f 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/RandomScoreFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/RandomScoreFunctionBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.index.query.functionscore; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.lucene.search.function.RandomScoreFunction; import org.opensearch.common.lucene.search.function.ScoreFunction; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/ScoreFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/ScoreFunctionBuilder.java index ff6f9bd55f9fa..c14c091bcdc48 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/ScoreFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/ScoreFunctionBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.index.query.functionscore; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.ScoreFunction; import org.opensearch.common.lucene.search.function.WeightFactorFunction; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreFunctionBuilder.java index 9edada8970981..a8c27d468a8f2 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreFunctionBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.index.query.functionscore; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.ScoreFunction; import org.opensearch.common.lucene.search.function.ScriptScoreFunction; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java index c27129a64a373..8d67a4be38dfb 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java @@ -35,8 +35,8 @@ import org.apache.lucene.search.Query; import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.ScriptScoreQuery; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/WeightBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/WeightBuilder.java index 8c503a9ded13c..018e475c4a513 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/WeightBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/WeightBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.index.query.functionscore; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.function.ScoreFunction; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java b/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java index d598b6d3baab1..3a67283a60180 100644 --- a/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java +++ b/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java @@ -31,9 +31,9 @@ package org.opensearch.index.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java b/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java index 9af23d5fa2f30..8a975020eadf5 100644 --- a/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java +++ b/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.refresh; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/reindex/AbstractBulkByScrollRequest.java b/server/src/main/java/org/opensearch/index/reindex/AbstractBulkByScrollRequest.java index 46883f8b274f2..1ed4df0fb4fe9 100644 --- a/server/src/main/java/org/opensearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/AbstractBulkByScrollRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.replication.ReplicationRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.search.Scroll; import org.opensearch.search.builder.SearchSourceBuilder; diff --git a/server/src/main/java/org/opensearch/index/reindex/AbstractBulkIndexByScrollRequest.java b/server/src/main/java/org/opensearch/index/reindex/AbstractBulkIndexByScrollRequest.java index f98ca036bd263..663f6136d2105 100644 --- a/server/src/main/java/org/opensearch/index/reindex/AbstractBulkIndexByScrollRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/AbstractBulkIndexByScrollRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.script.Script; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/index/reindex/BulkByScrollResponse.java b/server/src/main/java/org/opensearch/index/reindex/BulkByScrollResponse.java index 9060487f6364f..b96db441d6736 100644 --- a/server/src/main/java/org/opensearch/index/reindex/BulkByScrollResponse.java +++ b/server/src/main/java/org/opensearch/index/reindex/BulkByScrollResponse.java @@ -40,15 +40,15 @@ import org.opensearch.index.reindex.BulkByScrollTask.Status; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.index.reindex.ScrollableHitSource.SearchFailure; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; @@ -58,7 +58,7 @@ import static java.lang.Math.min; import static java.util.Objects.requireNonNull; import static org.opensearch.common.unit.TimeValue.timeValueNanos; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Response used for actions that index many documents using a scroll request. diff --git a/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java b/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java index 2fe0fe0259c33..040ee61ccd37d 100644 --- a/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java +++ b/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java @@ -37,9 +37,9 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ConstructingObjectParser; @@ -68,7 +68,7 @@ import static java.lang.Math.min; import static java.util.Collections.emptyList; import static org.opensearch.common.unit.TimeValue.timeValueNanos; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; /** diff --git a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java index 286aa2cc93045..998427c27c1f6 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java @@ -45,7 +45,7 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.client.Client; import org.opensearch.client.ParentTaskAssigningClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; diff --git a/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java index 0f5ff91efcd14..f3efa0040a0ea 100644 --- a/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java @@ -36,7 +36,7 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java index 90f7796bf1fdc..f6b4793f3b87a 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java @@ -36,10 +36,10 @@ import org.opensearch.action.CompositeIndicesRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/index/reindex/RemoteInfo.java b/server/src/main/java/org/opensearch/index/reindex/RemoteInfo.java index d5d69d5a6d3f8..78032c193c866 100644 --- a/server/src/main/java/org/opensearch/index/reindex/RemoteInfo.java +++ b/server/src/main/java/org/opensearch/index/reindex/RemoteInfo.java @@ -33,10 +33,10 @@ package org.opensearch.index.reindex; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java index 81560de78b336..a4047507fcde1 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java @@ -41,16 +41,16 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/reindex/UpdateByQueryRequest.java b/server/src/main/java/org/opensearch/index/reindex/UpdateByQueryRequest.java index 8ba31de8a48a1..6601fc3a7513f 100644 --- a/server/src/main/java/org/opensearch/index/reindex/UpdateByQueryRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/UpdateByQueryRequest.java @@ -35,8 +35,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java index 280381a7b6109..3f1161f0c5e03 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java @@ -17,7 +17,7 @@ import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Arrays; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index 919d34258a507..332b0d1698800 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -8,13 +8,13 @@ package org.opensearch.index.remote; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.MovingAverage; import org.opensearch.common.util.Streak; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/search/MatchQuery.java b/server/src/main/java/org/opensearch/index/search/MatchQuery.java index 91a4b456bfa2a..d38c9d365107b 100644 --- a/server/src/main/java/org/opensearch/index/search/MatchQuery.java +++ b/server/src/main/java/org/opensearch/index/search/MatchQuery.java @@ -58,9 +58,9 @@ import org.apache.lucene.util.QueryBuilder; import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause; diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java index 549c16e607506..4bae210f183c2 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java @@ -35,9 +35,9 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java index 3aa84976ab7fd..4ae693851c85f 100644 --- a/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java @@ -43,12 +43,12 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index 38288af4a57b6..08e899a40af7f 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -46,9 +46,9 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.WriteStateException; @@ -58,7 +58,7 @@ import org.opensearch.index.shard.AbstractIndexShardComponent; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationTimer; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java index 3f724704a7e67..5d4483e4a2930 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java @@ -33,9 +33,9 @@ package org.opensearch.index.seqno; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseActions.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseActions.java index 2ce0f1f524a43..88a4f5d8ab2b5 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseActions.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseActions.java @@ -44,13 +44,13 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lease.Releasable; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseAlreadyExistsException.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseAlreadyExistsException.java index f394b4b69affe..122b8279b08f9 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseAlreadyExistsException.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseAlreadyExistsException.java @@ -33,7 +33,7 @@ package org.opensearch.index.seqno; import org.opensearch.ResourceAlreadyExistsException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index 6de6aeaef5018..ec8d44846e30e 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -47,14 +47,14 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseInvalidRetainingSeqNoException.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseInvalidRetainingSeqNoException.java index b532846f57ac4..66af085974fa6 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseInvalidRetainingSeqNoException.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseInvalidRetainingSeqNoException.java @@ -33,7 +33,7 @@ package org.opensearch.index.seqno; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseNotFoundException.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseNotFoundException.java index 524778d0f9414..3a1a6e1d0025b 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseNotFoundException.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseNotFoundException.java @@ -33,7 +33,7 @@ package org.opensearch.index.seqno; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java index aba5fcb83c60a..a1ed615b83a14 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.seqno; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContent; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java index afcf5c6766194..0c50f9b06b2af 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java @@ -49,15 +49,15 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncer.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncer.java index bfe5fc75a0991..db3e8c4012ae5 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncer.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncer.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.common.inject.Inject; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java index 452bd2a1dd6a0..a764ca3e7a581 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java @@ -33,9 +33,9 @@ package org.opensearch.index.seqno; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java b/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java index 1114c91718b42..be7888ada2801 100644 --- a/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java +++ b/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.seqno; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/shard/AbstractIndexShardComponent.java b/server/src/main/java/org/opensearch/index/shard/AbstractIndexShardComponent.java index 2832c17ddc11a..5e973ff9acb8f 100644 --- a/server/src/main/java/org/opensearch/index/shard/AbstractIndexShardComponent.java +++ b/server/src/main/java/org/opensearch/index/shard/AbstractIndexShardComponent.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.logging.Loggers; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; /** diff --git a/server/src/main/java/org/opensearch/index/shard/DocsStats.java b/server/src/main/java/org/opensearch/index/shard/DocsStats.java index 34a1db1459ce5..83cc69752db2f 100644 --- a/server/src/main/java/org/opensearch/index/shard/DocsStats.java +++ b/server/src/main/java/org/opensearch/index/shard/DocsStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.store.StoreStats; diff --git a/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java b/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java index 35dfcf1a0f46d..ac21b7a9b1ab8 100644 --- a/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java +++ b/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java @@ -38,6 +38,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.FutureUtils; +import org.opensearch.core.index.shard.ShardId; import java.io.Closeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IllegalIndexShardStateException.java b/server/src/main/java/org/opensearch/index/shard/IllegalIndexShardStateException.java index bc9bc4a5a61fd..47ed149913914 100644 --- a/server/src/main/java/org/opensearch/index/shard/IllegalIndexShardStateException.java +++ b/server/src/main/java/org/opensearch/index/shard/IllegalIndexShardStateException.java @@ -33,9 +33,10 @@ package org.opensearch.index.shard; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java b/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java index 600bbc0af4cbc..9e23a84eac030 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java @@ -34,7 +34,8 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index bc90c42983617..8541a1f5e554b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -104,8 +104,9 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.gateway.WriteStateException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; @@ -186,7 +187,7 @@ import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.suggest.completion.CompletionStats; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardClosedException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardClosedException.java index e4452a377c167..8b4c9a188e00c 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardClosedException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardClosedException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardComponent.java b/server/src/main/java/org/opensearch/index/shard/IndexShardComponent.java index 66e0d74c47a72..cf9602ebd76d7 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardComponent.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardComponent.java @@ -32,6 +32,7 @@ package org.opensearch.index.shard; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; /** diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardNotRecoveringException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardNotRecoveringException.java index f75e1f2ac7bc5..75af4ef53e638 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardNotRecoveringException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardNotRecoveringException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardNotStartedException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardNotStartedException.java index cf33d30a339d5..7f37ae2a00873 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardNotStartedException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardNotStartedException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java b/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java index 389de85a04483..7b93d05fb322a 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java @@ -44,6 +44,7 @@ import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveringException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveringException.java index 11e93c978c7d1..9372ff222d6f0 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveringException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveringException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveryException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveryException.java index a4cf6164bae5d..0d8947b5bebb6 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveryException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardRecoveryException.java @@ -33,7 +33,8 @@ package org.opensearch.index.shard; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardRelocatedException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardRelocatedException.java index 3d63dba938d5c..5176b23edcf82 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardRelocatedException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardRelocatedException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardStartedException.java b/server/src/main/java/org/opensearch/index/shard/IndexShardStartedException.java index fd7985a32bd58..a724952e37707 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardStartedException.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardStartedException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexingOperationListener.java b/server/src/main/java/org/opensearch/index/shard/IndexingOperationListener.java index ed6e9f82ad064..ccc4cd336cff7 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexingOperationListener.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexingOperationListener.java @@ -33,6 +33,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.engine.Engine; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java index d40878674ac13..f45417a20036e 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java @@ -33,9 +33,9 @@ package org.opensearch.index.shard; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java b/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java index f0b87ee3c2b41..d7e15dd3e40f5 100644 --- a/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java @@ -34,6 +34,7 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.metrics.MeanMetric; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.engine.Engine; import java.util.concurrent.TimeUnit; diff --git a/server/src/main/java/org/opensearch/index/shard/LocalShardSnapshot.java b/server/src/main/java/org/opensearch/index/shard/LocalShardSnapshot.java index 82ec4c72e3d0c..a98fc2a3b4145 100644 --- a/server/src/main/java/org/opensearch/index/shard/LocalShardSnapshot.java +++ b/server/src/main/java/org/opensearch/index/shard/LocalShardSnapshot.java @@ -41,7 +41,7 @@ import org.apache.lucene.store.NoLockFactory; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.concurrent.GatedCloseable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.Engine; import org.opensearch.index.store.Store; diff --git a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java index 5786c25031647..4f0affb3035ec 100644 --- a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java +++ b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java @@ -41,13 +41,14 @@ import org.opensearch.action.resync.TransportResyncReplicationAction; import org.opensearch.common.Strings; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/shard/PrimaryShardClosedException.java b/server/src/main/java/org/opensearch/index/shard/PrimaryShardClosedException.java index d1b2bf9079289..34adbd67ac9f2 100644 --- a/server/src/main/java/org/opensearch/index/shard/PrimaryShardClosedException.java +++ b/server/src/main/java/org/opensearch/index/shard/PrimaryShardClosedException.java @@ -7,7 +7,8 @@ import java.io.IOException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; /** * Exception to indicate failures are caused due to the closure of the primary diff --git a/server/src/main/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommand.java b/server/src/main/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommand.java index bd4cbd6573961..6bc7828a06d44 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommand.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommand.java @@ -62,11 +62,12 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeMetadata; import org.opensearch.gateway.PersistedClusterStateService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/shard/ShardNotFoundException.java b/server/src/main/java/org/opensearch/index/shard/ShardNotFoundException.java index c97f392f54a5a..c68c89b96cb4f 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardNotFoundException.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardNotFoundException.java @@ -33,7 +33,8 @@ package org.opensearch.index.shard; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/ShardNotInPrimaryModeException.java b/server/src/main/java/org/opensearch/index/shard/ShardNotInPrimaryModeException.java index cb03424e027e4..412dee8a7abcf 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardNotInPrimaryModeException.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardNotInPrimaryModeException.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/shard/ShardPath.java b/server/src/main/java/org/opensearch/index/shard/ShardPath.java index 39486ec00a36a..64c949a0c537d 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardPath.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardPath.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.util.Strings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; diff --git a/server/src/main/java/org/opensearch/index/shard/ShardUtils.java b/server/src/main/java/org/opensearch/index/shard/ShardUtils.java index 3d51fa6a05357..8e5688b26f93d 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardUtils.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardUtils.java @@ -37,6 +37,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.lucene.index.OpenSearchLeafReader; +import org.opensearch.core.index.shard.ShardId; /** * Utility class for shard operations diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index da4e9113143af..d4e779c83644f 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -53,7 +53,8 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineException; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreException.java b/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreException.java index af2fedc36c60d..1cb31f215f4a5 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreException.java +++ b/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreException.java @@ -33,8 +33,8 @@ package org.opensearch.index.snapshots; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreFailedException.java b/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreFailedException.java index 07cc1367e72b5..50bf5a2e6aeef 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreFailedException.java +++ b/server/src/main/java/org/opensearch/index/snapshots/IndexShardRestoreFailedException.java @@ -32,8 +32,8 @@ package org.opensearch.index.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index d295027a857cd..2d49b153c39f4 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -43,8 +43,8 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.store.StoreFileMetadata; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index df931875c5dd0..710f73b3b1fa6 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -37,7 +37,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java b/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java index 4738e1c1cbeec..9e8c8d29c2058 100644 --- a/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java +++ b/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java @@ -8,9 +8,9 @@ package org.opensearch.index.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java index d9102c26fee64..8f4f0b661ed33 100644 --- a/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java +++ b/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/stats/ShardIndexingPressureStats.java b/server/src/main/java/org/opensearch/index/stats/ShardIndexingPressureStats.java index 35c003b49bfd6..1635c560dce3f 100644 --- a/server/src/main/java/org/opensearch/index/stats/ShardIndexingPressureStats.java +++ b/server/src/main/java/org/opensearch/index/stats/ShardIndexingPressureStats.java @@ -8,13 +8,13 @@ package org.opensearch.index.stats; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 46e5e627ef415..8967100d4faf0 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -67,11 +67,11 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.store.ByteArrayIndexInput; @@ -92,7 +92,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.AbstractIndexShardComponent; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/index/store/StoreFileMetadata.java b/server/src/main/java/org/opensearch/index/store/StoreFileMetadata.java index 30bab979b46c5..8415b65d838b7 100644 --- a/server/src/main/java/org/opensearch/index/store/StoreFileMetadata.java +++ b/server/src/main/java/org/opensearch/index/store/StoreFileMetadata.java @@ -35,9 +35,9 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/store/StoreStats.java b/server/src/main/java/org/opensearch/index/store/StoreStats.java index 192086947c120..ba36e6b527031 100644 --- a/server/src/main/java/org/opensearch/index/store/StoreStats.java +++ b/server/src/main/java/org/opensearch/index/store/StoreStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.store; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java index 8cc2c821a97f5..6d09bc062ab0e 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java @@ -14,11 +14,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexEventListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.cluster.IndicesClusterStateService; diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java index 3c2c8fab7a30d..46a81adc1ab45 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java @@ -8,9 +8,9 @@ package org.opensearch.index.store.remote.filecache; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java index 53734460a2d65..3314b2e8d2c8c 100644 --- a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java @@ -46,7 +46,7 @@ import org.opensearch.action.termvectors.TermVectorsRequest; import org.opensearch.action.termvectors.TermVectorsResponse; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java index aaa54bfdf7c2d..ea817c4b0932a 100644 --- a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java @@ -32,7 +32,7 @@ package org.opensearch.index.translog; -import org.opensearch.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.index.seqno.SequenceNumbers; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java index f299da0c1ac1e..8c9ccc3b487df 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java +++ b/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java @@ -33,8 +33,8 @@ package org.opensearch.index.translog; import org.apache.lucene.store.BufferedChecksum; -import org.opensearch.common.io.stream.FilterStreamInput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.FilterStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.EOFException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java b/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java index 0efffccfb8a7f..eaf9ac0893f1b 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java +++ b/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java @@ -33,7 +33,7 @@ package org.opensearch.index.translog; import org.apache.lucene.store.BufferedChecksum; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.zip.CRC32; diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java index 9b3240823f368..4d728da8a394d 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java @@ -16,7 +16,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.LifecycleAware; import org.opensearch.index.seqno.LocalCheckpointTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.listener.TranslogEventListener; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java index dd5593b6d79cd..3e6a8e69edfbb 100644 --- a/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/NoOpTranslogManager.java @@ -9,7 +9,7 @@ package org.opensearch.index.translog; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.stream.Stream; diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 9e027b9765bbc..87fadce1d834c 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -16,7 +16,7 @@ import org.opensearch.core.util.FileSystemUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.index.translog.transfer.FileTransferTracker; import org.opensearch.index.translog.transfer.TransferSnapshot; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index a9acf76f4834d..f0177f3588db1 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -37,11 +37,11 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ReleasableLock; @@ -59,7 +59,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.AbstractIndexShardComponent; import org.opensearch.index.shard.IndexShardComponent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.Closeable; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java index 1d01dc854be37..2862accfedc43 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java @@ -36,7 +36,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.nio.file.Path; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogCorruptedException.java b/server/src/main/java/org/opensearch/index/translog/TranslogCorruptedException.java index cf3d07c0ef910..4cc7844b1ced0 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogCorruptedException.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogCorruptedException.java @@ -33,7 +33,7 @@ package org.opensearch.index.translog; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogException.java b/server/src/main/java/org/opensearch/index/translog/TranslogException.java index ffa964c13976e..d7a8d649543d6 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogException.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogException.java @@ -33,8 +33,8 @@ package org.opensearch.index.translog; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index 8067cccb772a2..1090a994bf6ad 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -40,9 +40,9 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.Channels; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.EOFException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java index 7774db16f3178..c740e29963c4d 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.translog; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index e7b08b1dda3d2..1fd3a9c3341f6 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -39,8 +39,8 @@ import org.apache.lucene.util.BytesRefIterator; import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; @@ -53,7 +53,7 @@ import org.opensearch.common.lease.Releasables; import org.opensearch.core.Assertions; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.ByteArrayOutputStream; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/index/translog/TruncatedTranslogException.java b/server/src/main/java/org/opensearch/index/translog/TruncatedTranslogException.java index 3660e2c835e11..1b7f38e03817e 100644 --- a/server/src/main/java/org/opensearch/index/translog/TruncatedTranslogException.java +++ b/server/src/main/java/org/opensearch/index/translog/TruncatedTranslogException.java @@ -32,7 +32,7 @@ package org.opensearch.index.translog; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java index 0f7340a6a5a8b..9080bc483138b 100644 --- a/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java @@ -11,7 +11,7 @@ import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.index.engine.LifecycleAware; import org.opensearch.index.seqno.LocalCheckpointTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.listener.TranslogEventListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/translog/listener/CompositeTranslogEventListener.java b/server/src/main/java/org/opensearch/index/translog/listener/CompositeTranslogEventListener.java index b738fa0feea59..9216616d5e9ab 100644 --- a/server/src/main/java/org/opensearch/index/translog/listener/CompositeTranslogEventListener.java +++ b/server/src/main/java/org/opensearch/index/translog/listener/CompositeTranslogEventListener.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ExceptionsHelper; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.TranslogException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/FileTransferTracker.java b/server/src/main/java/org/opensearch/index/translog/transfer/FileTransferTracker.java index 1909164bd821a..406533561a798 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/FileTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/FileTransferTracker.java @@ -8,7 +8,7 @@ package org.opensearch.index.translog.transfer; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import org.opensearch.index.translog.transfer.listener.FileTransferListener; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 0c63a7ffe4cce..e2bb5f74df234 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -18,12 +18,12 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.stream.write.WritePriority; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.store.ByteArrayIndexInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.transfer.listener.TranslogTransferListener; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/warmer/ShardIndexWarmerService.java b/server/src/main/java/org/opensearch/index/warmer/ShardIndexWarmerService.java index c690ed8c1c385..ff8f31297d47d 100644 --- a/server/src/main/java/org/opensearch/index/warmer/ShardIndexWarmerService.java +++ b/server/src/main/java/org/opensearch/index/warmer/ShardIndexWarmerService.java @@ -37,7 +37,7 @@ import org.opensearch.common.metrics.MeanMetric; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.AbstractIndexShardComponent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.concurrent.TimeUnit; diff --git a/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java b/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java index 6e373a50dd89b..a0fd32b43796b 100644 --- a/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java +++ b/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java @@ -32,9 +32,9 @@ package org.opensearch.index.warmer; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java index 9bcebc2a73cb8..19fcbdd7b0bc7 100644 --- a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java +++ b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java @@ -32,7 +32,7 @@ package org.opensearch.indices; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.cache.RemovalNotification; import org.opensearch.common.cache.RemovalReason; import org.opensearch.index.cache.request.ShardRequestCache; diff --git a/server/src/main/java/org/opensearch/indices/AliasFilterParsingException.java b/server/src/main/java/org/opensearch/indices/AliasFilterParsingException.java index 494824b11bafe..1f8fc5f1dd7d0 100644 --- a/server/src/main/java/org/opensearch/indices/AliasFilterParsingException.java +++ b/server/src/main/java/org/opensearch/indices/AliasFilterParsingException.java @@ -33,8 +33,8 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/IndexClosedException.java b/server/src/main/java/org/opensearch/indices/IndexClosedException.java index 64ffcf173f46f..cf5c46f9294b3 100644 --- a/server/src/main/java/org/opensearch/indices/IndexClosedException.java +++ b/server/src/main/java/org/opensearch/indices/IndexClosedException.java @@ -33,9 +33,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/IndexCreationException.java b/server/src/main/java/org/opensearch/indices/IndexCreationException.java index 2eca5c8f0a5d9..cd604176441ef 100644 --- a/server/src/main/java/org/opensearch/indices/IndexCreationException.java +++ b/server/src/main/java/org/opensearch/indices/IndexCreationException.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/IndexPrimaryShardNotAllocatedException.java b/server/src/main/java/org/opensearch/indices/IndexPrimaryShardNotAllocatedException.java index a646ae53ecc5a..68bd3202231f8 100644 --- a/server/src/main/java/org/opensearch/indices/IndexPrimaryShardNotAllocatedException.java +++ b/server/src/main/java/org/opensearch/indices/IndexPrimaryShardNotAllocatedException.java @@ -33,9 +33,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/IndexTemplateMissingException.java b/server/src/main/java/org/opensearch/indices/IndexTemplateMissingException.java index 78060c1b3a072..fc55c9d82eddd 100644 --- a/server/src/main/java/org/opensearch/indices/IndexTemplateMissingException.java +++ b/server/src/main/java/org/opensearch/indices/IndexTemplateMissingException.java @@ -32,9 +32,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java b/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java index 611d348fb6791..9a87f1c26fa29 100644 --- a/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java +++ b/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java @@ -47,7 +47,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.IndexingOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.Scheduler.Cancellable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.threadpool.ThreadPool.Names; diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java index b868f6aa35aee..9d2eef5f67a86 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesModule.java +++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java @@ -40,8 +40,8 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.ParseField; import org.opensearch.common.inject.AbstractModule; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.mapper.BinaryFieldMapper; diff --git a/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java b/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java index 2669da3f417c3..6ebed17437074 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java @@ -50,7 +50,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.index.cache.query.QueryCacheStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.Closeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index 151131e719dc1..a0208fab9cbef 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -41,7 +41,7 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.common.CheckedSupplier; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.cache.Cache; import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.cache.CacheLoader; diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 60e90a21b6309..0ffad8ce65b7a 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -61,13 +61,13 @@ import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Nullable; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -95,7 +95,7 @@ import org.opensearch.env.ShardLockObtainFailedException; import org.opensearch.gateway.MetaStateService; import org.opensearch.gateway.MetadataStateFormat; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; @@ -131,7 +131,7 @@ import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.IndexingStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.remote.filecache.FileCacheCleaner; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; diff --git a/server/src/main/java/org/opensearch/indices/InvalidAliasNameException.java b/server/src/main/java/org/opensearch/indices/InvalidAliasNameException.java index 47184fc608bea..4ccb778c7010f 100644 --- a/server/src/main/java/org/opensearch/indices/InvalidAliasNameException.java +++ b/server/src/main/java/org/opensearch/indices/InvalidAliasNameException.java @@ -33,9 +33,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/InvalidIndexNameException.java b/server/src/main/java/org/opensearch/indices/InvalidIndexNameException.java index 631aaa937cf63..0d26a927bade5 100644 --- a/server/src/main/java/org/opensearch/indices/InvalidIndexNameException.java +++ b/server/src/main/java/org/opensearch/indices/InvalidIndexNameException.java @@ -33,9 +33,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/InvalidIndexTemplateException.java b/server/src/main/java/org/opensearch/indices/InvalidIndexTemplateException.java index 17a7334e6867a..e5cce1a9c4487 100644 --- a/server/src/main/java/org/opensearch/indices/InvalidIndexTemplateException.java +++ b/server/src/main/java/org/opensearch/indices/InvalidIndexTemplateException.java @@ -32,9 +32,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/InvalidTypeNameException.java b/server/src/main/java/org/opensearch/indices/InvalidTypeNameException.java index 1e33488b6fb8a..48c9595e789bd 100644 --- a/server/src/main/java/org/opensearch/indices/InvalidTypeNameException.java +++ b/server/src/main/java/org/opensearch/indices/InvalidTypeNameException.java @@ -32,9 +32,9 @@ package org.opensearch.indices; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.mapper.MapperException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java index 7f7b9fd133882..482cd07543051 100644 --- a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java +++ b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java @@ -36,12 +36,12 @@ import org.opensearch.action.admin.indices.stats.IndexShardStats; import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.engine.SegmentsStats; diff --git a/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java b/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java index 63a0cad402061..c86137177e750 100644 --- a/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java +++ b/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java @@ -39,7 +39,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Arrays; import java.util.Optional; diff --git a/server/src/main/java/org/opensearch/indices/SystemIndices.java b/server/src/main/java/org/opensearch/indices/SystemIndices.java index 3d5df507348fb..a85e938c61b7a 100644 --- a/server/src/main/java/org/opensearch/indices/SystemIndices.java +++ b/server/src/main/java/org/opensearch/indices/SystemIndices.java @@ -42,7 +42,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; import org.opensearch.common.regex.Regex; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.tasks.TaskResultsService; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/indices/TermsLookup.java b/server/src/main/java/org/opensearch/indices/TermsLookup.java index 9346654e9c4fb..37533c0809d7a 100644 --- a/server/src/main/java/org/opensearch/indices/TermsLookup.java +++ b/server/src/main/java/org/opensearch/indices/TermsLookup.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/indices/TypeMissingException.java b/server/src/main/java/org/opensearch/indices/TypeMissingException.java index a57e62bf2a76e..c74572f6900c7 100644 --- a/server/src/main/java/org/opensearch/indices/TypeMissingException.java +++ b/server/src/main/java/org/opensearch/indices/TypeMissingException.java @@ -33,9 +33,9 @@ package org.opensearch.indices; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java b/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java index ef7f947f986ce..83f3f9532948f 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java +++ b/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java @@ -32,9 +32,9 @@ package org.opensearch.indices.breaker; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java b/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java index 1675088aad492..94e63acd10648 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java +++ b/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java @@ -32,9 +32,9 @@ package org.opensearch.indices.breaker; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java index e4b251914fa0b..b3fc070d62e58 100644 --- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java @@ -61,7 +61,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.env.ShardLockObtainFailedException; import org.opensearch.gateway.GatewayService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexComponent; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; @@ -75,7 +75,7 @@ import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.PeerRecoverySourceService; diff --git a/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java b/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java index 45f78155589b3..dc18097419904 100644 --- a/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -53,11 +53,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldDataCache; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/DelayRecoveryException.java b/server/src/main/java/org/opensearch/indices/recovery/DelayRecoveryException.java index bd88dcc236fb4..687bed05f0bef 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/DelayRecoveryException.java +++ b/server/src/main/java/org/opensearch/indices/recovery/DelayRecoveryException.java @@ -33,7 +33,7 @@ package org.opensearch.indices.recovery; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/FileChunkRequest.java b/server/src/main/java/org/opensearch/indices/recovery/FileChunkRequest.java index 3594495224481..a40245cddda38 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/FileChunkRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/FileChunkRequest.java @@ -33,11 +33,11 @@ package org.opensearch.indices.recovery; import org.apache.lucene.util.Version; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.StoreFileMetadata; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/FileChunkWriter.java b/server/src/main/java/org/opensearch/indices/recovery/FileChunkWriter.java index f1cc7b8dd1d89..c7021a0e969b9 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/FileChunkWriter.java +++ b/server/src/main/java/org/opensearch/indices/recovery/FileChunkWriter.java @@ -9,7 +9,7 @@ package org.opensearch.indices.recovery; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.store.StoreFileMetadata; /** diff --git a/server/src/main/java/org/opensearch/indices/recovery/ForceSyncRequest.java b/server/src/main/java/org/opensearch/indices/recovery/ForceSyncRequest.java index 2600097fd0f2a..09b13b2e5451d 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/ForceSyncRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/ForceSyncRequest.java @@ -8,9 +8,9 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java b/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java index c27852b27960b..0110e8b6d162a 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java +++ b/server/src/main/java/org/opensearch/indices/recovery/MultiFileWriter.java @@ -37,7 +37,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryNotFound.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryNotFound.java index 478dabba7aef0..ed6246ec73fed 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryNotFound.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryNotFound.java @@ -33,8 +33,8 @@ package org.opensearch.indices.recovery; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java index 8bea14a1a1c86..66b5b3f8b7535 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java @@ -53,7 +53,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index 17af529a700d9..a289c8f8a04b7 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -47,7 +47,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; @@ -59,7 +59,7 @@ import org.opensearch.index.shard.IllegalIndexShardStateException; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java index fe7bba1769463..8d60b44c2c6bd 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java @@ -34,10 +34,10 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryCleanFilesRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryCleanFilesRequest.java index d346ec5c975f4..50997b2d8949b 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryCleanFilesRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryCleanFilesRequest.java @@ -32,9 +32,9 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryFailedException.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryFailedException.java index 12393ab12c95d..73916d0238419 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryFailedException.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryFailedException.java @@ -34,8 +34,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.common.ReplicationFailedException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryFilesInfoRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryFilesInfoRequest.java index dccadfcf49cf1..d0fefe3d51f33 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryFilesInfoRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryFilesInfoRequest.java @@ -32,9 +32,9 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java index 446fb78958db4..f57743281ab9d 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryFinalizeRecoveryRequest.java @@ -32,9 +32,9 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryHandoffPrimaryContextRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryHandoffPrimaryContextRequest.java index 4e47668d82656..3662d9106cacc 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryHandoffPrimaryContextRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryHandoffPrimaryContextRequest.java @@ -32,10 +32,10 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.seqno.ReplicationTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java index 68979fa4b69bc..9cc34dce7be82 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryPrepareForTranslogOperationsRequest.java @@ -32,9 +32,9 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryResponse.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryResponse.java index 3a9bbf9da8928..8af69b1786e38 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryResponse.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryResponse.java @@ -32,8 +32,8 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java index 0da82db3d3e8d..3418e82bc25b0 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java @@ -36,14 +36,14 @@ import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationTimer; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java index 7466ab5c1ca86..1eba50ffcf172 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java @@ -41,7 +41,7 @@ import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.util.CancellableThreads; import org.opensearch.index.engine.Engine; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsRequest.java index 32560bc211669..e8c24fb10e02f 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsRequest.java @@ -32,10 +32,10 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.seqno.RetentionLeases; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsResponse.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsResponse.java index b4ea8bd3e89b6..b623d382b415f 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsResponse.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTranslogOperationsResponse.java @@ -32,8 +32,8 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportResponse; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTransportRequest.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTransportRequest.java index dc5beec87c2b7..3280ce2425fdf 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTransportRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTransportRequest.java @@ -32,8 +32,8 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/ReestablishRecoveryRequest.java b/server/src/main/java/org/opensearch/indices/recovery/ReestablishRecoveryRequest.java index 179e16f5f0cf6..cdada2b458723 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/ReestablishRecoveryRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/ReestablishRecoveryRequest.java @@ -32,9 +32,9 @@ package org.opensearch.indices.recovery; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java index a8dd083ba838e..cdc62350b4aa5 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java @@ -36,11 +36,11 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLeases; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.translog.Translog; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java index 77ffd75d0148a..04bee5586cbab 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java @@ -15,7 +15,7 @@ import org.opensearch.action.support.RetryableAction; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.ConcurrentCollections; diff --git a/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java b/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java index 24e6504cde59d..60aa413cbca4f 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java @@ -33,10 +33,10 @@ package org.opensearch.indices.recovery; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.transport.TransportRequest; diff --git a/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoRequest.java b/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoRequest.java index 188a4c1e40fa7..6120b5302eb35 100644 --- a/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoRequest.java +++ b/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoRequest.java @@ -9,8 +9,8 @@ package org.opensearch.indices.replication; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.SegmentReplicationTransportRequest; diff --git a/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java b/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java index 48c2dfd30f589..0155883f34552 100644 --- a/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java +++ b/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java @@ -8,8 +8,8 @@ package org.opensearch.indices.replication; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.transport.TransportResponse; diff --git a/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesRequest.java b/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesRequest.java index daad33ed93f28..9fd90837cb1a5 100644 --- a/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesRequest.java +++ b/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesRequest.java @@ -9,8 +9,8 @@ package org.opensearch.indices.replication; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.SegmentReplicationTransportRequest; diff --git a/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesResponse.java b/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesResponse.java index 6dc7e293b2c31..89d50a17464a6 100644 --- a/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesResponse.java +++ b/server/src/main/java/org/opensearch/indices/replication/GetSegmentFilesResponse.java @@ -8,8 +8,8 @@ package org.opensearch.indices.replication; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.transport.TransportResponse; diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java index f32887175d4f3..e0e356f1531e1 100644 --- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java +++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java @@ -17,7 +17,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.FileChunkWriter; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java index 5455be2a69799..b1a6cab2ba57a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java b/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java index b3909a3c0f8df..ed171927c4600 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java @@ -11,9 +11,9 @@ import org.apache.lucene.store.RateLimiter; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java index 44029f0c89cb0..aec7908ef1165 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java @@ -17,8 +17,8 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.store.InputStreamIndexInput; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java index 238e316c3b585..a2ec8bfe54ecc 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java @@ -12,7 +12,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java index 79186deeeaf0f..9d8b30209ab80 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java @@ -24,7 +24,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RetryableTransportClient; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 226ccbaf01afa..c5d526251d9e7 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -10,9 +10,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index f59a7c2368689..2e0f5a8c0ad1f 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -22,7 +22,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.util.CancellableThreads; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 467f499056345..7c35c4f07598e 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -24,7 +24,7 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.indices.recovery.ForceSyncRequest; diff --git a/server/src/main/java/org/opensearch/indices/replication/UpdateVisibleCheckpointRequest.java b/server/src/main/java/org/opensearch/indices/replication/UpdateVisibleCheckpointRequest.java index 2674adf711406..118643f1928f2 100644 --- a/server/src/main/java/org/opensearch/indices/replication/UpdateVisibleCheckpointRequest.java +++ b/server/src/main/java/org/opensearch/indices/replication/UpdateVisibleCheckpointRequest.java @@ -9,9 +9,9 @@ package org.opensearch.indices.replication; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.SegmentReplicationTransportRequest; diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 9e54b210fea04..e77f9e12212fc 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -21,7 +21,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointRequest.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointRequest.java index 740fd3bccb7c4..f9ad5733bee62 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointRequest.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointRequest.java @@ -9,8 +9,8 @@ package org.opensearch.indices.replication.checkpoint; import org.opensearch.action.support.replication.ReplicationRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java index 32521fb0cd944..da01023ace47c 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java @@ -10,11 +10,11 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java index c65ef27969154..431e35906702a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationCollection.java @@ -40,7 +40,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationFailedException.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationFailedException.java index 23ad4d0e096b5..f8f08dbe7a452 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationFailedException.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationFailedException.java @@ -10,9 +10,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java index ce39f69ddb3a5..9a9690581ed24 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java @@ -9,9 +9,9 @@ package org.opensearch.indices.replication.common; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java index 344a4040be119..0c96a87715014 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java @@ -16,13 +16,13 @@ import org.opensearch.action.support.ChannelActionListener; import org.opensearch.common.CheckedFunction; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.FileChunkRequest; diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java index eeb0ba8bb2c00..59c4e24ce3efa 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java @@ -8,9 +8,9 @@ package org.opensearch.indices.replication.common; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/replication/common/SegmentReplicationTransportRequest.java b/server/src/main/java/org/opensearch/indices/replication/common/SegmentReplicationTransportRequest.java index 09b14fb1b5333..667009d2b7479 100644 --- a/server/src/main/java/org/opensearch/indices/replication/common/SegmentReplicationTransportRequest.java +++ b/server/src/main/java/org/opensearch/indices/replication/common/SegmentReplicationTransportRequest.java @@ -9,8 +9,8 @@ package org.opensearch.indices.replication.common; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java index 5c34aba6632b2..eaaf5198fba94 100644 --- a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java @@ -50,8 +50,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; @@ -61,7 +61,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java index bdb0d99fa93b0..22c5c923e6322 100644 --- a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java +++ b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java @@ -48,9 +48,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.env.NodeEnvironment; @@ -60,7 +60,7 @@ import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; diff --git a/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java index b212e67a59434..894d699aee7f5 100644 --- a/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java @@ -37,7 +37,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/ingest/IngestInfo.java b/server/src/main/java/org/opensearch/ingest/IngestInfo.java index d886bd4615c92..122aa4ab92002 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestInfo.java +++ b/server/src/main/java/org/opensearch/ingest/IngestInfo.java @@ -32,8 +32,8 @@ package org.opensearch.ingest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/ingest/IngestMetadata.java b/server/src/main/java/org/opensearch/ingest/IngestMetadata.java index a8391b8c1002f..293aa8a00eca6 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestMetadata.java +++ b/server/src/main/java/org/opensearch/ingest/IngestMetadata.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/ingest/IngestProcessorException.java b/server/src/main/java/org/opensearch/ingest/IngestProcessorException.java index eccc81cf62459..c7920d73b87aa 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestProcessorException.java +++ b/server/src/main/java/org/opensearch/ingest/IngestProcessorException.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/ingest/IngestStats.java b/server/src/main/java/org/opensearch/ingest/IngestStats.java index ac06d779bdf2d..558712316bb2c 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestStats.java +++ b/server/src/main/java/org/opensearch/ingest/IngestStats.java @@ -32,11 +32,11 @@ package org.opensearch.ingest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.common.metrics.OperationStats; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java index 07d0c31fc6703..08a3708ae60ae 100644 --- a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.Diff; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/ingest/ProcessorInfo.java b/server/src/main/java/org/opensearch/ingest/ProcessorInfo.java index 9008567dac955..276e0ac234a8e 100644 --- a/server/src/main/java/org/opensearch/ingest/ProcessorInfo.java +++ b/server/src/main/java/org/opensearch/ingest/ProcessorInfo.java @@ -32,9 +32,9 @@ package org.opensearch.ingest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java index 1aa7e00ed86c1..cfa29436f24a5 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java index a58f640d9a856..1acdf00ca6af0 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java @@ -36,9 +36,9 @@ import org.opensearch.common.Booleans; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.PathUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java index 3f029f3d0aaf8..5451821a565cd 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java @@ -32,9 +32,9 @@ package org.opensearch.monitor.jvm; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/monitor/os/OsInfo.java b/server/src/main/java/org/opensearch/monitor/os/OsInfo.java index 5890e878c49f4..cc19e4a82aed8 100644 --- a/server/src/main/java/org/opensearch/monitor/os/OsInfo.java +++ b/server/src/main/java/org/opensearch/monitor/os/OsInfo.java @@ -32,8 +32,8 @@ package org.opensearch.monitor.os; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/monitor/os/OsStats.java b/server/src/main/java/org/opensearch/monitor/os/OsStats.java index da5fe5531eb9f..c684bf10c4a03 100644 --- a/server/src/main/java/org/opensearch/monitor/os/OsStats.java +++ b/server/src/main/java/org/opensearch/monitor/os/OsStats.java @@ -34,9 +34,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/monitor/process/ProcessInfo.java b/server/src/main/java/org/opensearch/monitor/process/ProcessInfo.java index f1fcbea722a84..8664deee63d3e 100644 --- a/server/src/main/java/org/opensearch/monitor/process/ProcessInfo.java +++ b/server/src/main/java/org/opensearch/monitor/process/ProcessInfo.java @@ -32,8 +32,8 @@ package org.opensearch.monitor.process; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java b/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java index b0a9784665315..de49c07a5600f 100644 --- a/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java +++ b/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java @@ -32,9 +32,9 @@ package org.opensearch.monitor.process; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/node/AdaptiveSelectionStats.java b/server/src/main/java/org/opensearch/node/AdaptiveSelectionStats.java index df5c7b93bdc2d..9d06d0c2cfe20 100644 --- a/server/src/main/java/org/opensearch/node/AdaptiveSelectionStats.java +++ b/server/src/main/java/org/opensearch/node/AdaptiveSelectionStats.java @@ -32,9 +32,9 @@ package org.opensearch.node; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index aecf5659de7fe..ee89964dde810 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -118,7 +118,7 @@ import org.opensearch.common.inject.Key; import org.opensearch.common.inject.Module; import org.opensearch.common.inject.ModulesBuilder; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.logging.HeaderWarning; import org.opensearch.common.logging.NodeAndClusterIdStateListener; diff --git a/server/src/main/java/org/opensearch/node/NodeClosedException.java b/server/src/main/java/org/opensearch/node/NodeClosedException.java index a62a8a4a23684..d9e8dbe8466e4 100644 --- a/server/src/main/java/org/opensearch/node/NodeClosedException.java +++ b/server/src/main/java/org/opensearch/node/NodeClosedException.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/node/ReportingService.java b/server/src/main/java/org/opensearch/node/ReportingService.java index 064d90bae9699..969652e215e5e 100644 --- a/server/src/main/java/org/opensearch/node/ReportingService.java +++ b/server/src/main/java/org/opensearch/node/ReportingService.java @@ -32,7 +32,7 @@ package org.opensearch.node; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; /** diff --git a/server/src/main/java/org/opensearch/node/ResponseCollectorService.java b/server/src/main/java/org/opensearch/node/ResponseCollectorService.java index fd246a4ccb25e..3b73ec8f9622d 100644 --- a/server/src/main/java/org/opensearch/node/ResponseCollectorService.java +++ b/server/src/main/java/org/opensearch/node/ResponseCollectorService.java @@ -37,9 +37,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ExponentiallyWeightedMovingAverage; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java index d036457ccae89..9ef1749c077c9 100644 --- a/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/persistent/PersistentTaskParams.java b/server/src/main/java/org/opensearch/persistent/PersistentTaskParams.java index 796105b740846..b71d7f61d0106 100644 --- a/server/src/main/java/org/opensearch/persistent/PersistentTaskParams.java +++ b/server/src/main/java/org/opensearch/persistent/PersistentTaskParams.java @@ -33,7 +33,7 @@ package org.opensearch.persistent; import org.opensearch.cluster.ClusterState; -import org.opensearch.common.io.stream.VersionedNamedWriteable; +import org.opensearch.core.common.io.stream.VersionedNamedWriteable; import org.opensearch.core.xcontent.ToXContentObject; /** diff --git a/server/src/main/java/org/opensearch/persistent/PersistentTaskResponse.java b/server/src/main/java/org/opensearch/persistent/PersistentTaskResponse.java index 7abc81d72d698..b0fe789badb5e 100644 --- a/server/src/main/java/org/opensearch/persistent/PersistentTaskResponse.java +++ b/server/src/main/java/org/opensearch/persistent/PersistentTaskResponse.java @@ -32,8 +32,8 @@ package org.opensearch.persistent; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.persistent.PersistentTasksCustomMetadata.PersistentTask; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/persistent/PersistentTaskState.java b/server/src/main/java/org/opensearch/persistent/PersistentTaskState.java index 4f2eceab7eba1..954c5c3e1938c 100644 --- a/server/src/main/java/org/opensearch/persistent/PersistentTaskState.java +++ b/server/src/main/java/org/opensearch/persistent/PersistentTaskState.java @@ -31,7 +31,7 @@ package org.opensearch.persistent; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContentObject; /** diff --git a/server/src/main/java/org/opensearch/persistent/PersistentTasksCustomMetadata.java b/server/src/main/java/org/opensearch/persistent/PersistentTasksCustomMetadata.java index 9a49fb0bc33a6..48a169a8da961 100644 --- a/server/src/main/java/org/opensearch/persistent/PersistentTasksCustomMetadata.java +++ b/server/src/main/java/org/opensearch/persistent/PersistentTasksCustomMetadata.java @@ -42,9 +42,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/opensearch/persistent/PersistentTasksNodeService.java index 5b7e83695046d..0900cf2e19254 100644 --- a/server/src/main/java/org/opensearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/opensearch/persistent/PersistentTasksNodeService.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterStateListener; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.gateway.GatewayService; diff --git a/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java index d07dcc23056d8..b9d9ea8f51fca 100644 --- a/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java index 4864cf3c23b50..b607bd7441858 100644 --- a/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java @@ -46,8 +46,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java index acbb62373ab60..32c8961120663 100644 --- a/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java index 4132ae35ad60c..8ea8333f4851a 100644 --- a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java @@ -36,7 +36,7 @@ import java.util.Map; import java.util.function.Supplier; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/plugins/Plugin.java b/server/src/main/java/org/opensearch/plugins/Plugin.java index aff36a9884203..02a46f44ef23b 100644 --- a/server/src/main/java/org/opensearch/plugins/Plugin.java +++ b/server/src/main/java/org/opensearch/plugins/Plugin.java @@ -43,8 +43,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.component.LifecycleComponent; import org.opensearch.common.inject.Module; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.SettingUpgrader; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/plugins/PluginInfo.java b/server/src/main/java/org/opensearch/plugins/PluginInfo.java index 09a6d878662bb..8936f9eba7a7f 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginInfo.java +++ b/server/src/main/java/org/opensearch/plugins/PluginInfo.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.bootstrap.JarHell; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index 247b1db8e1da6..d55cec18f2c54 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -37,9 +37,9 @@ import org.apache.lucene.search.Sort; import org.opensearch.common.CheckedFunction; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.search.function.ScoreFunction; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ContextParser; diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index 764f36df6d337..462d105dc0c68 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -43,7 +43,7 @@ import org.opensearch.common.component.Lifecycle; import org.opensearch.common.component.LifecycleListener; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index db65616b3cd16..eb6fb051957c7 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -32,13 +32,13 @@ package org.opensearch.repositories; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 4367b4c65851e..8a712b0a0c098 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -44,7 +44,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.component.LifecycleComponent; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java b/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java index dc796d4c0e4ab..185735f39c24c 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java @@ -33,9 +33,9 @@ import org.opensearch.common.Strings; import org.opensearch.common.blobstore.DeleteResult; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryData.java b/server/src/main/java/org/opensearch/repositories/RepositoryData.java index 9f12e5f00b2bc..90d7b404d089a 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryData.java @@ -39,7 +39,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotState; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryException.java b/server/src/main/java/org/opensearch/repositories/RepositoryException.java index 6606bf36a6136..a3e8cc9c3744e 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryException.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryException.java @@ -33,8 +33,8 @@ package org.opensearch.repositories; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java b/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java index f52dfc320d6f0..40dfd8cc77529 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java @@ -34,9 +34,9 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryMissingException.java b/server/src/main/java/org/opensearch/repositories/RepositoryMissingException.java index c84c25127237b..c39e03e5bb656 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryMissingException.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryMissingException.java @@ -32,8 +32,8 @@ package org.opensearch.repositories; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryShardId.java b/server/src/main/java/org/opensearch/repositories/RepositoryShardId.java index 1dd4bf438e01e..4799f36156a73 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryShardId.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryShardId.java @@ -32,9 +32,9 @@ package org.opensearch.repositories; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryStats.java b/server/src/main/java/org/opensearch/repositories/RepositoryStats.java index 006258859822f..efd5d6f8560b6 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryStats.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryStats.java @@ -32,9 +32,9 @@ package org.opensearch.repositories; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java index cdeb6ae53d203..2357b572ed3d0 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java @@ -33,9 +33,9 @@ package org.opensearch.repositories; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable;; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable;; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryVerificationException.java b/server/src/main/java/org/opensearch/repositories/RepositoryVerificationException.java index aaf06f4dbcd7f..485635b3b618c 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryVerificationException.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryVerificationException.java @@ -32,8 +32,8 @@ package org.opensearch.repositories; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/opensearch/repositories/VerifyNodeRepositoryAction.java index 7a6e4291e3f72..ff5ffdbfe1e3e 100644 --- a/server/src/main/java/org/opensearch/repositories/VerifyNodeRepositoryAction.java +++ b/server/src/main/java/org/opensearch/repositories/VerifyNodeRepositoryAction.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.EmptyTransportResponseHandler; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 398d78392ae17..ae8080da73fa8 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -75,14 +75,14 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.compress.Compressor; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.compress.CompressorType; -import org.opensearch.common.compress.NotXContentException; +import org.opensearch.core.common.compress.NotXContentException; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.store.InputStreamIndexInput; @@ -102,9 +102,9 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardRestoreFailedException; -import org.opensearch.index.snapshots.IndexShardSnapshotFailedException; +import org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java index b0dba60e2188f..cb3d779ece4a9 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.CheckedFunction; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.Compressor; import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/FileRestoreContext.java b/server/src/main/java/org/opensearch/repositories/blobstore/FileRestoreContext.java index 8217e73c01a3c..080485f750173 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/FileRestoreContext.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/FileRestoreContext.java @@ -37,7 +37,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.util.iterable.Iterables; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardRestoreFailedException; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.opensearch.index.snapshots.blobstore.SnapshotFiles; diff --git a/server/src/main/java/org/opensearch/rest/BaseRestHandler.java b/server/src/main/java/org/opensearch/rest/BaseRestHandler.java index 8693626ef7faa..e18a594236fc8 100644 --- a/server/src/main/java/org/opensearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/opensearch/rest/BaseRestHandler.java @@ -44,6 +44,7 @@ import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugins.ActionPlugin; import org.opensearch.rest.action.admin.cluster.RestNodesUsageAction; diff --git a/server/src/main/java/org/opensearch/rest/BytesRestResponse.java b/server/src/main/java/org/opensearch/rest/BytesRestResponse.java index 3bb094bcd4ba0..c8a4e566e8e4d 100644 --- a/server/src/main/java/org/opensearch/rest/BytesRestResponse.java +++ b/server/src/main/java/org/opensearch/rest/BytesRestResponse.java @@ -39,8 +39,9 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchStatusException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -48,7 +49,7 @@ import java.io.IOException; import static java.util.Collections.singletonMap; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * REST response in bytes diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index e4b10e0babda4..87d1542102f91 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -39,13 +39,14 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.Nullable; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.path.PathTrie; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentType; @@ -75,11 +76,11 @@ import static org.opensearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; import static org.opensearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.INTERNAL_SERVER_ERROR; -import static org.opensearch.rest.RestStatus.METHOD_NOT_ALLOWED; -import static org.opensearch.rest.RestStatus.NOT_ACCEPTABLE; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.opensearch.core.rest.RestStatus.METHOD_NOT_ALLOWED; +import static org.opensearch.core.rest.RestStatus.NOT_ACCEPTABLE; +import static org.opensearch.core.rest.RestStatus.OK; /** * OpenSearch REST controller diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 36961ed3e20da..653720391d6a3 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -38,8 +38,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/rest/RestRequestFilter.java b/server/src/main/java/org/opensearch/rest/RestRequestFilter.java index f24c5a15c4304..dbb968401f9eb 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequestFilter.java +++ b/server/src/main/java/org/opensearch/rest/RestRequestFilter.java @@ -33,7 +33,7 @@ package org.opensearch.rest; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/rest/RestResponse.java b/server/src/main/java/org/opensearch/rest/RestResponse.java index fd08f8cf96fd8..eb4ad99f03cfc 100644 --- a/server/src/main/java/org/opensearch/rest/RestResponse.java +++ b/server/src/main/java/org/opensearch/rest/RestResponse.java @@ -33,8 +33,9 @@ package org.opensearch.rest; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lease.Releasable; +import org.opensearch.core.rest.RestStatus; import java.util.ArrayList; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/rest/action/RestActions.java b/server/src/main/java/org/opensearch/rest/action/RestActions.java index 17d07849a4731..7508871d3ab75 100644 --- a/server/src/main/java/org/opensearch/rest/action/RestActions.java +++ b/server/src/main/java/org/opensearch/rest/action/RestActions.java @@ -34,12 +34,12 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.FailedNodeException; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.ToXContent; @@ -54,7 +54,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/rest/action/RestMainAction.java b/server/src/main/java/org/opensearch/rest/action/RestMainAction.java index ae8cdcb2f8ef6..54bd09d8e534e 100644 --- a/server/src/main/java/org/opensearch/rest/action/RestMainAction.java +++ b/server/src/main/java/org/opensearch/rest/action/RestMainAction.java @@ -41,7 +41,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/rest/action/RestStatusToXContentListener.java b/server/src/main/java/org/opensearch/rest/action/RestStatusToXContentListener.java index e44fa3df9e0f9..74b4409f90c70 100644 --- a/server/src/main/java/org/opensearch/rest/action/RestStatusToXContentListener.java +++ b/server/src/main/java/org/opensearch/rest/action/RestStatusToXContentListener.java @@ -36,7 +36,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.function.Function; diff --git a/server/src/main/java/org/opensearch/rest/action/RestToXContentListener.java b/server/src/main/java/org/opensearch/rest/action/RestToXContentListener.java index 5d4d86563dd49..be7414c33c2cb 100644 --- a/server/src/main/java/org/opensearch/rest/action/RestToXContentListener.java +++ b/server/src/main/java/org/opensearch/rest/action/RestToXContentListener.java @@ -38,7 +38,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; /** * A REST based action listener that assumes the response is of type {@link ToXContent} and automatically diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java index 051528c8b4080..d8c8db039899d 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java @@ -42,7 +42,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java index bf505cfdece4e..bc49f59c7de7d 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java @@ -48,7 +48,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStateAction.java index 24f8e30f49ea8..7e9bbf4993d65 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStateAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStateAction.java @@ -49,7 +49,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestListTasksAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestListTasksAction.java index b50c23edb7169..cdca1ae975ae7 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestListTasksAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestListTasksAction.java @@ -45,7 +45,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import org.opensearch.rest.action.RestToXContentListener; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java index 443b903a54fb3..b5a0626ca74e9 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java @@ -42,7 +42,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestResponseListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesUsageAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesUsageAction.java index 3b9789cfc73e0..858990874de2e 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesUsageAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesUsageAction.java @@ -41,7 +41,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActions; import org.opensearch.rest.action.RestBuilderListener; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java index f17ac0f48e750..28381af769a51 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java @@ -33,7 +33,7 @@ import org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BaseRestHandler; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java index 4233c2bd4c0f6..72b2b025e8f5b 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.core.ParseField; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; @@ -46,7 +46,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestRequestFilter; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActions; import org.opensearch.rest.action.RestBuilderListener; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java index 0cf0b76a25e23..c74bfff344e68 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java @@ -38,7 +38,7 @@ import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -46,7 +46,7 @@ import static java.util.Collections.singletonList; import static org.opensearch.rest.RestRequest.Method.DELETE; -import static org.opensearch.rest.RestStatus.ACCEPTED; +import static org.opensearch.core.rest.RestStatus.ACCEPTED; /** * Transport action to delete dangling index diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java index f2405afdab834..76a554f52cedb 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java @@ -34,7 +34,7 @@ import static java.util.Collections.singletonList; import static org.opensearch.rest.RestRequest.Method.POST; -import static org.opensearch.rest.RestStatus.ACCEPTED; +import static org.opensearch.core.rest.RestStatus.ACCEPTED; import java.io.IOException; import java.util.List; @@ -45,7 +45,7 @@ import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; /** diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/AliasesNotFoundException.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/AliasesNotFoundException.java index 595b6a4b2ed70..e85df8451c836 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/AliasesNotFoundException.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/AliasesNotFoundException.java @@ -32,7 +32,7 @@ package org.opensearch.rest.action.admin.indices; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetAliasesAction.java index 15605a32e4189..10674ba253641 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -46,7 +46,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComponentTemplateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComponentTemplateAction.java index cc3192e143397..172d5f071ebb8 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComponentTemplateAction.java @@ -38,7 +38,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -48,8 +48,8 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.HEAD; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; /** * Transport action to get component template diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java index c4725324b44d1..86dcaaf5914dd 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java @@ -38,7 +38,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -48,8 +48,8 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.HEAD; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; /** * Transport action to get composable index template diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetFieldMappingAction.java index ca48b4de2f609..9a1c07748e974 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -46,7 +46,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.io.IOException; @@ -56,8 +56,8 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; /** * Transport action to get field mapping diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 2a86d356372a6..e409546d676ae 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -40,7 +40,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -51,8 +51,8 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.HEAD; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; /** * The REST handler for get template and head template APIs. diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetMappingAction.java index b89d660ad8e7c..9f95541aab332 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestGetMappingAction.java @@ -47,7 +47,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActionListener; import org.opensearch.rest.action.RestBuilderListener; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndicesShardStoresAction.java index e80fd1519d507..eff3cfe601ec1 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndicesShardStoresAction.java @@ -51,7 +51,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; /** * Rest action for {@link IndicesShardStoresAction} diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestRefreshAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestRefreshAction.java index 2d26d09e09e6e..90e06a2446057 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestRefreshAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestRefreshAction.java @@ -39,7 +39,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestSyncedFlushAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestSyncedFlushAction.java index 99d5d706f60c8..9cb4a90eeb834 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestSyncedFlushAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestSyncedFlushAction.java @@ -44,7 +44,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestToXContentListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestValidateQueryAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestValidateQueryAction.java index 3e064bc0a170a..211322e0d7419 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestValidateQueryAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestValidateQueryAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BaseRestHandler; @@ -54,7 +54,7 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; /** * Transport action to validate a query diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java index abc08b9318ac1..d1d16bd1af17e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java @@ -35,11 +35,11 @@ import org.opensearch.common.Table; import org.opensearch.common.io.Streams; import org.opensearch.common.io.UTF8StreamWriter; -import org.opensearch.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.BytesStream; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatAction.java index b0fd6dd03fe4e..1652f24bdf574 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatAction.java @@ -36,7 +36,7 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index cb4b2ca281348..556928d4685a0 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -36,7 +36,7 @@ import org.opensearch.common.Table; import org.opensearch.common.io.Streams; import org.opensearch.common.io.UTF8StreamWriter; -import org.opensearch.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.BytesStream; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.SizeValue; @@ -48,7 +48,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestGetAction.java index 2abb3135319e5..bd7dd59b5d0ff 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestGetAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestGetAction.java @@ -39,7 +39,7 @@ import org.opensearch.index.VersionType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestActions; import org.opensearch.rest.action.RestToXContentListener; import org.opensearch.search.fetch.subphase.FetchSourceContext; @@ -51,8 +51,8 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.HEAD; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; /** * Transport action to get a document diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java index 54d03890b74ee..7247067755721 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.rest.BaseRestHandler; @@ -56,7 +56,7 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.HEAD; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; /** * The REST handler for get source and head source APIs. diff --git a/server/src/main/java/org/opensearch/rest/action/ingest/RestPutPipelineAction.java b/server/src/main/java/org/opensearch/rest/action/ingest/RestPutPipelineAction.java index 40b2db4bafc45..192ab58045473 100644 --- a/server/src/main/java/org/opensearch/rest/action/ingest/RestPutPipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/ingest/RestPutPipelineAction.java @@ -34,7 +34,7 @@ import org.opensearch.action.ingest.PutPipelineRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java index eb193edda3373..ec8cc2c00e65a 100644 --- a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java @@ -34,7 +34,7 @@ import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BaseRestHandler; diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java index f5348aba0ad6f..dedcabded9071 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java @@ -19,7 +19,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java index 190818ce08d0d..189b1e93e6444 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java @@ -38,9 +38,9 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; import org.opensearch.common.CheckedBiConsumer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java index a2bb061e52c32..2f398a6bea9ff 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java @@ -10,7 +10,7 @@ import org.opensearch.action.search.PutSearchPipelineRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BaseRestHandler; diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java index 34a9ad6ed39eb..744bdcb3a3acc 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java @@ -40,7 +40,7 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Booleans; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/script/GeneralScriptException.java b/server/src/main/java/org/opensearch/script/GeneralScriptException.java index b52a242b5de92..ffc0c4d27d841 100644 --- a/server/src/main/java/org/opensearch/script/GeneralScriptException.java +++ b/server/src/main/java/org/opensearch/script/GeneralScriptException.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/script/Script.java b/server/src/main/java/org/opensearch/script/Script.java index 7f6c41575ccb8..37555dbec4dbc 100644 --- a/server/src/main/java/org/opensearch/script/Script.java +++ b/server/src/main/java/org/opensearch/script/Script.java @@ -33,13 +33,13 @@ package org.opensearch.script; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.AbstractObjectParser; diff --git a/server/src/main/java/org/opensearch/script/ScriptCacheStats.java b/server/src/main/java/org/opensearch/script/ScriptCacheStats.java index 7445e40baceef..be7a1c9542a60 100644 --- a/server/src/main/java/org/opensearch/script/ScriptCacheStats.java +++ b/server/src/main/java/org/opensearch/script/ScriptCacheStats.java @@ -32,9 +32,9 @@ package org.opensearch.script; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/script/ScriptContextInfo.java b/server/src/main/java/org/opensearch/script/ScriptContextInfo.java index ccf7dd3a7a412..d3a64c207e0ce 100644 --- a/server/src/main/java/org/opensearch/script/ScriptContextInfo.java +++ b/server/src/main/java/org/opensearch/script/ScriptContextInfo.java @@ -33,9 +33,9 @@ package org.opensearch.script; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/script/ScriptContextStats.java b/server/src/main/java/org/opensearch/script/ScriptContextStats.java index 84c179b98fb2c..16f4a7e313326 100644 --- a/server/src/main/java/org/opensearch/script/ScriptContextStats.java +++ b/server/src/main/java/org/opensearch/script/ScriptContextStats.java @@ -32,9 +32,9 @@ package org.opensearch.script; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/script/ScriptException.java b/server/src/main/java/org/opensearch/script/ScriptException.java index 361626ef39fba..c2ca58be9507d 100644 --- a/server/src/main/java/org/opensearch/script/ScriptException.java +++ b/server/src/main/java/org/opensearch/script/ScriptException.java @@ -34,12 +34,12 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java b/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java index a144d98a7e3a5..64332910147de 100644 --- a/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java +++ b/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java @@ -34,9 +34,9 @@ import org.opensearch.core.ParseField; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/script/ScriptMetadata.java b/server/src/main/java/org/opensearch/script/ScriptMetadata.java index 170a74582393a..7036e9ad4d53e 100644 --- a/server/src/main/java/org/opensearch/script/ScriptMetadata.java +++ b/server/src/main/java/org/opensearch/script/ScriptMetadata.java @@ -38,10 +38,10 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/script/ScriptStats.java b/server/src/main/java/org/opensearch/script/ScriptStats.java index 4836a2a32ace7..aeb3645242799 100644 --- a/server/src/main/java/org/opensearch/script/ScriptStats.java +++ b/server/src/main/java/org/opensearch/script/ScriptStats.java @@ -32,9 +32,9 @@ package org.opensearch.script; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/script/ScriptType.java b/server/src/main/java/org/opensearch/script/ScriptType.java index 079f788b29df9..5f505c781bd0a 100644 --- a/server/src/main/java/org/opensearch/script/ScriptType.java +++ b/server/src/main/java/org/opensearch/script/ScriptType.java @@ -33,9 +33,9 @@ package org.opensearch.script; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/script/StoredScriptSource.java b/server/src/main/java/org/opensearch/script/StoredScriptSource.java index 49ee0c42ccab7..1d6f2b8e4ccea 100644 --- a/server/src/main/java/org/opensearch/script/StoredScriptSource.java +++ b/server/src/main/java/org/opensearch/script/StoredScriptSource.java @@ -36,12 +36,12 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.Diff; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java index 1a378d6e1ff46..de0fb813eb652 100644 --- a/server/src/main/java/org/opensearch/search/DocValueFormat.java +++ b/server/src/main/java/org/opensearch/search/DocValueFormat.java @@ -35,9 +35,9 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Numbers; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.joda.Joda; import org.opensearch.common.joda.JodaDateFormatter; import org.opensearch.common.network.InetAddresses; diff --git a/server/src/main/java/org/opensearch/search/MultiValueMode.java b/server/src/main/java/org/opensearch/search/MultiValueMode.java index 687eddf7f4667..0fbd41f062710 100644 --- a/server/src/main/java/org/opensearch/search/MultiValueMode.java +++ b/server/src/main/java/org/opensearch/search/MultiValueMode.java @@ -42,9 +42,9 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.fielddata.AbstractBinaryDocValues; import org.opensearch.index.fielddata.AbstractNumericDocValues; import org.opensearch.index.fielddata.AbstractSortedDocValues; diff --git a/server/src/main/java/org/opensearch/search/RescoreDocIds.java b/server/src/main/java/org/opensearch/search/RescoreDocIds.java index d93d590fcb817..880f34a924f2a 100644 --- a/server/src/main/java/org/opensearch/search/RescoreDocIds.java +++ b/server/src/main/java/org/opensearch/search/RescoreDocIds.java @@ -32,9 +32,9 @@ package org.opensearch.search; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/search/Scroll.java b/server/src/main/java/org/opensearch/search/Scroll.java index 747d7b73aee8f..562979b98ec7d 100644 --- a/server/src/main/java/org/opensearch/search/Scroll.java +++ b/server/src/main/java/org/opensearch/search/Scroll.java @@ -32,9 +32,9 @@ package org.opensearch.search; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/SearchContextMissingException.java b/server/src/main/java/org/opensearch/search/SearchContextMissingException.java index a74dc412f280c..93bbe7e9ce64f 100644 --- a/server/src/main/java/org/opensearch/search/SearchContextMissingException.java +++ b/server/src/main/java/org/opensearch/search/SearchContextMissingException.java @@ -33,9 +33,9 @@ package org.opensearch.search; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.internal.ShardSearchContextId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/SearchException.java b/server/src/main/java/org/opensearch/search/SearchException.java index 93cccd0d73070..657e39d663513 100644 --- a/server/src/main/java/org/opensearch/search/SearchException.java +++ b/server/src/main/java/org/opensearch/search/SearchException.java @@ -34,8 +34,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/SearchExtBuilder.java b/server/src/main/java/org/opensearch/search/SearchExtBuilder.java index 2f004dabccceb..4d86c6c2e2277 100644 --- a/server/src/main/java/org/opensearch/search/SearchExtBuilder.java +++ b/server/src/main/java/org/opensearch/search/SearchExtBuilder.java @@ -33,10 +33,10 @@ package org.opensearch.search; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.plugins.SearchPlugin; import org.opensearch.plugins.SearchPlugin.SearchExtSpec; diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 3ad13c7e8b63d..4524359359d6d 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -37,15 +37,15 @@ import org.opensearch.Version; import org.opensearch.action.OriginalIndices; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; @@ -62,7 +62,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.SourceFieldMapper; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.search.lookup.SourceLookup; import org.opensearch.transport.RemoteClusterAware; @@ -82,8 +82,8 @@ import static java.util.Collections.unmodifiableMap; import static org.opensearch.common.lucene.Lucene.readExplanation; import static org.opensearch.common.lucene.Lucene.writeExplanation; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg; diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 7571cd4137404..c16a724eb0c75 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -52,7 +52,7 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Encapsulates the results of a search operation diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index aeb1d8325b1b8..d401a3143338b 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -37,9 +37,9 @@ import org.opensearch.common.Nullable; import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.ShapesAvailability; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; diff --git a/server/src/main/java/org/opensearch/search/SearchParseException.java b/server/src/main/java/org/opensearch/search/SearchParseException.java index e5f269e65de9d..ea21162dc7aea 100644 --- a/server/src/main/java/org/opensearch/search/SearchParseException.java +++ b/server/src/main/java/org/opensearch/search/SearchParseException.java @@ -33,11 +33,11 @@ package org.opensearch.search; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentLocation; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java index fd07e4b10939b..1b4cebbe91a3e 100644 --- a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java +++ b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java @@ -33,8 +33,8 @@ package org.opensearch.search; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.fetch.FetchSearchResult; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.internal.ShardSearchRequest; diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 9daad9112e473..c9c70ed52c376 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -55,8 +55,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.component.AbstractLifecycleComponent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -70,7 +70,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; @@ -85,7 +85,7 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.SearchOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; diff --git a/server/src/main/java/org/opensearch/search/SearchShardTarget.java b/server/src/main/java/org/opensearch/search/SearchShardTarget.java index 3d72ba6804efc..882ba8e069c2e 100644 --- a/server/src/main/java/org/opensearch/search/SearchShardTarget.java +++ b/server/src/main/java/org/opensearch/search/SearchShardTarget.java @@ -34,11 +34,11 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.text.Text; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.transport.RemoteClusterAware; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/SearchSortValues.java b/server/src/main/java/org/opensearch/search/SearchSortValues.java index 5dac54bac79c5..398d593d5db0d 100644 --- a/server/src/main/java/org/opensearch/search/SearchSortValues.java +++ b/server/src/main/java/org/opensearch/search/SearchSortValues.java @@ -33,14 +33,14 @@ package org.opensearch.search; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.SearchHit.Fields; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/SearchSortValuesAndFormats.java b/server/src/main/java/org/opensearch/search/SearchSortValuesAndFormats.java index de5fdca592546..1013a2bc4b36c 100644 --- a/server/src/main/java/org/opensearch/search/SearchSortValuesAndFormats.java +++ b/server/src/main/java/org/opensearch/search/SearchSortValuesAndFormats.java @@ -33,9 +33,9 @@ package org.opensearch.search; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/AbstractAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/AbstractAggregationBuilder.java index 19ca7c5e91ccd..d3f17a58874bc 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AbstractAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AbstractAggregationBuilder.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilder.java index 5b7d3631c296e..ab5cdb83d0153 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilder.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationExecutionException.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationExecutionException.java index 989783b11da3c..c211b9cf5f344 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationExecutionException.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationExecutionException.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationInitializationException.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationInitializationException.java index 87fcfece07064..1159023cb6be2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationInitializationException.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationInitializationException.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/Aggregations.java b/server/src/main/java/org/opensearch/search/aggregations/Aggregations.java index 635b18f32deaf..575b8bfdf6e94 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/Aggregations.java +++ b/server/src/main/java/org/opensearch/search/aggregations/Aggregations.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -49,7 +49,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; -import static org.opensearch.common.xcontent.XContentParserUtils.parseTypedKeysObject; +import static org.opensearch.core.xcontent.XContentParserUtils.parseTypedKeysObject; /** * Represents a set of {@link Aggregation}s diff --git a/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java b/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java index 4d9b973e4550f..e157a8fe5ff8a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lease.Releasable; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java index e57776fd78c49..04fa34466e0ff 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java @@ -32,11 +32,11 @@ package org.opensearch.search.aggregations; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.SuggestingErrorOnUnknown; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedObjectNotFoundException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/BucketOrder.java b/server/src/main/java/org/opensearch/search/aggregations/BucketOrder.java index be2c1da5bed03..637829ea5127f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/BucketOrder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/BucketOrder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; diff --git a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java index 3c9dd2ef0e049..b7577fb647be5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java index 228360b872042..e4d64e1e8517c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java +++ b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.aggregations.InternalAggregation.ReduceContext; import org.opensearch.search.aggregations.pipeline.PipelineAggregator; import org.opensearch.search.aggregations.pipeline.SiblingPipelineAggregator; diff --git a/server/src/main/java/org/opensearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/InternalMultiBucketAggregation.java index c191507cf991f..1d9df65fee92d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/InternalMultiBucketAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/InternalMultiBucketAggregation.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import org.opensearch.search.aggregations.bucket.SingleBucketAggregation; import org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; diff --git a/server/src/main/java/org/opensearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/opensearch/search/aggregations/InternalOrder.java index 651f44d859cf6..091f1797d8f78 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/InternalOrder.java @@ -31,9 +31,9 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.util.Comparators; import org.opensearch.core.xcontent.XContent; diff --git a/server/src/main/java/org/opensearch/search/aggregations/InvalidAggregationPathException.java b/server/src/main/java/org/opensearch/search/aggregations/InvalidAggregationPathException.java index 1222e3904ea5e..08ea0553affd1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/InvalidAggregationPathException.java +++ b/server/src/main/java/org/opensearch/search/aggregations/InvalidAggregationPathException.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java index efa83f824d079..83a94afbe8e02 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java +++ b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java @@ -33,12 +33,12 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.aggregations.bucket.BucketsAggregator; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/ParsedMultiBucketAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/ParsedMultiBucketAggregation.java index 456b5bdd2707f..5cbe0c9f3eeee 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/ParsedMultiBucketAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/ParsedMultiBucketAggregation.java @@ -37,7 +37,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import java.io.IOException; @@ -45,7 +45,7 @@ import java.util.List; import java.util.function.Supplier; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A multi-bucket agg that has been parsed diff --git a/server/src/main/java/org/opensearch/search/aggregations/PipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/PipelineAggregationBuilder.java index eb6475fff86ff..e71ea1054be9e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/PipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/PipelineAggregationBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ValidateActions; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.index.query.QueryRewriteContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/InternalSingleBucketAggregation.java index 733185fafb3b6..03fade2edb392 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/InternalSingleBucketAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/InternalSingleBucketAggregation.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java index f7b835e229af2..6f946dc50b9b4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/ParsedSingleBucketAggregation.java @@ -33,7 +33,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedAggregation; @@ -43,7 +43,7 @@ import java.util.Collections; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A base class for all the single bucket aggregations. diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java index 5f57b0e2de3b0..743d0023364fa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.adjacency; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java index 7abc6ae1771f0..5b8f38b9873ea 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.core.xcontent.ObjectParser.NamedObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java index 5158b33b1d00b..d96b1f00e488e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.adjacency; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index a03c93dbd9d70..5e8791441d83a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.composite; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeKey.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeKey.java index 94b32c8c02ff0..5ddeb22d33a6f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeKey.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeKey.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java index 1805cbbbb76cc..8a4504d677823 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java index caa57c52c8698..a56250f57780c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.composite; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.AbstractObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContent.Params; @@ -47,7 +47,7 @@ import java.io.IOException; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER; import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_AGGREGATION_TYPE; import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_BYTE_CODE; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java index ddb64371eda16..3ef1488ab1c2f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java @@ -35,8 +35,8 @@ import org.apache.lucene.index.IndexReader; import org.opensearch.core.ParseField; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java index 00cd869f90ddc..a6872e5278588 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.composite; import org.apache.lucene.index.IndexReader; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java index e7277739eb1f0..27619044d8995 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.composite; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java index 52ea0b00f3f72..21b9419e8f93a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java @@ -34,8 +34,8 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java index 2e8deb19bbbe3..586283fca6b63 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.filter; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java index 0c44afebe4415..125cbf5d533c7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.filter; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregator.java index 84e0218d8eaab..01777940009d5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregator.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilter.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilter.java index e3cf61a4ef816..61fe979be7910 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilter.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilter.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.filter; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilters.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilters.java index e48fc5d3cdd31..104dab01d90fe 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilters.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/InternalFilters.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.filter; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/ParsedFilters.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/ParsedFilters.java index 96937e1471305..71e7f0ecbb730 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/ParsedFilters.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/ParsedFilters.java @@ -35,7 +35,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; @@ -46,7 +46,7 @@ import java.util.List; import java.util.Map; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A filters agg result parsed between nodes diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java index 317a97d563e99..5404dfe91f9d1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.global; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/global/InternalGlobal.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/global/InternalGlobal.java index 68e4f1d4e2c10..feed446b23e21 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/global/InternalGlobal.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/global/InternalGlobal.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.global; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java index b5d1c33d2e7e4..882b18364f0a7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java @@ -34,9 +34,9 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java index b6b0c2cec5b68..556ff1838cc66 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java @@ -37,7 +37,7 @@ import org.apache.lucene.util.CollectionUtil; import org.opensearch.common.Rounding; import org.opensearch.common.Rounding.Prepared; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.IntArray; import org.opensearch.common.util.LongArray; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java index 9fc7f2f10eb14..6e7fc962a692a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java index c49a2ac496768..9e8509e257bc6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java index 509ab4d308107..bb325a737721c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java @@ -35,9 +35,9 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Rounding; import org.opensearch.common.Rounding.DateTimeUnit; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java index da788f7de7d91..69c70ed1bf7fd 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.InstantiatingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java index 799420867dd70..a117cb1931868 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java index fb83ae2e489d9..0866d26526761 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java @@ -33,8 +33,8 @@ import org.apache.lucene.util.PriorityQueue; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalDateHistogram.java index 03b4d1e143b71..e0b6010c6c3e8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalDateHistogram.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalDateHistogram.java @@ -34,8 +34,8 @@ import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.PriorityQueue; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalHistogram.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalHistogram.java index 71a1adc3f9095..a27c689127ac9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalHistogram.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalHistogram.java @@ -33,8 +33,8 @@ import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.PriorityQueue; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java index bd72ff04fa7b3..52f689eb7c229 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.apache.lucene.util.PriorityQueue; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java index 901de22f08e95..478be3633cb21 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java @@ -35,9 +35,9 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.core.ParseField; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/ParsedVariableWidthHistogram.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/ParsedVariableWidthHistogram.java index 6acdf877d7f8e..8b59f8af722ff 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/ParsedVariableWidthHistogram.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/ParsedVariableWidthHistogram.java @@ -35,7 +35,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; @@ -44,7 +44,7 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A variable width histogram result parsed between nodes diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java index 0a8fba48cf254..66ff6baed66ec 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/InternalMissing.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/InternalMissing.java index 7cc86483f0b56..be71bbdeb2c04 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/InternalMissing.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/InternalMissing.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.missing; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java index e542740066bca..77e01d4e32093 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.missing; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingOrder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingOrder.java index b0b987065c647..dcdc838f31eec 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingOrder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/missing/MissingOrder.java @@ -9,9 +9,9 @@ package org.opensearch.search.aggregations.bucket.missing; import org.opensearch.common.inject.Provider; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalNested.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalNested.java index c3346c66d1733..f4cdbf19f37cf 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalNested.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalNested.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.nested; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalReverseNested.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalReverseNested.java index 95b4714bdba9b..5852d8b1cd0dd 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalReverseNested.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/InternalReverseNested.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.nested; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java index e2501fc0853af..0d98fdbf15f78 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.bucket.nested; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.ObjectMapper; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java index c2358a71c3edc..e798ec9402cb9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.bucket.nested; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.ObjectMapper; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeBuilder.java index 63dfbf3cdf573..df44f5130cc85 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/AbstractRangeBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.range; import org.apache.lucene.util.InPlaceMergeSorter; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.bucket.range.RangeAggregator.Range; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java index fbd5636433462..7668aa78b4bc0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.range; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java index 609938cc621ad..e0d506d95a75d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java @@ -33,17 +33,17 @@ package org.opensearch.search.aggregations.bucket.range; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoDistance; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories.Builder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalBinaryRange.java index ec35a2f683e5d..b98747b89f450 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalBinaryRange.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalBinaryRange.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.range; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalDateRange.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalDateRange.java index 03908dcaca4bb..146aa728a88b7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalDateRange.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalDateRange.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.range; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalGeoDistance.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalGeoDistance.java index 4d5e9cbfb7e49..5d58a139b5023 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalGeoDistance.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalGeoDistance.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.range; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.support.CoreValuesSourceType; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalRange.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalRange.java index 47e7f96f2f415..2abe196416e52 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalRange.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/InternalRange.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.range; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java index ff0dc0fb2ec66..f011f950cf289 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java @@ -34,10 +34,10 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.network.InetAddresses; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedBinaryRange.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedBinaryRange.java index 610d6295dacf4..9b687022f5302 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedBinaryRange.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedBinaryRange.java @@ -35,7 +35,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; @@ -44,7 +44,7 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A binary range agg result parsed between nodes diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedRange.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedRange.java index 029f1df478509..1922debd40271 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedRange.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/ParsedRange.java @@ -36,7 +36,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; @@ -46,7 +46,7 @@ import java.util.List; import java.util.function.Supplier; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A range agg result parsed between nodes diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilder.java index 28762926e68a8..c027587aacbd1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilder.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.range; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java index a33dee25f4faf..b56b817b8177b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/range/RangeAggregator.java @@ -34,9 +34,9 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ScoreMode; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java index 5b0e56f20acea..c905d99507658 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.sampler; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/InternalSampler.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/InternalSampler.java index 8b6f4d32bed3c..b05c501b55dc1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/InternalSampler.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/InternalSampler.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.sampler; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.bucket.InternalSingleBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java index 581d1a739e88e..951329fa81fcc 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.bucket.sampler; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/UnmappedSampler.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/UnmappedSampler.java index 3527788a5dc1d..a22d20addb49d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/UnmappedSampler.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/UnmappedSampler.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.bucket.sampler; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java index 56113c5d2ee16..0b76c302801af 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/DoubleTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 996ff2fea8b1a..e0a22435b8f48 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -40,7 +40,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.PriorityQueue; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.LongArray; import org.opensearch.common.util.LongHash; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java index 0617ed02359b5..8f0e50f2348ae 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java @@ -49,9 +49,9 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java index 011a6293d7bf3..ff742d189c7c7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; import org.opensearch.common.Randomness; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.SetBackedScalingCuckooFilter; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedSignificantTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedSignificantTerms.java index 26e919c3f80c1..97a95b8df840b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedSignificantTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedSignificantTerms.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java index f8de28a348bd6..f5e92fec8195d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMappedTerms.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java index 599b7736d24ff..fc84f35385d5c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java @@ -9,8 +9,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.AggregationExecutionException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalRareTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalRareTerms.java index 5888043d15856..3bcbe2716651a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalRareTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalRareTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.SetBackedScalingCuckooFilter; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java index 789be3c1c9d9e..6104d2193f6cd 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java index 7795a462d6043..9a80155eea51c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java @@ -33,8 +33,8 @@ import org.apache.lucene.util.PriorityQueue; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.AggregationExecutionException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongRareTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongRareTerms.java index 27e715768348d..18a97b1cbdb9a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongRareTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongRareTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.SetBackedScalingCuckooFilter; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java index d5dfdb5f81dff..67aa80d0a9879 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/LongTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java index 197e03ae3fa1d..8c7b552e3b9de 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java @@ -9,8 +9,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java index fccb9c3af5986..9d99c0b90a075 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java @@ -17,10 +17,10 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Numbers; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedRareTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedRareTerms.java index 53f2e62d407f3..83702b2856b9a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedRareTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedRareTerms.java @@ -37,7 +37,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedSignificantTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedSignificantTerms.java index 21cf0e860d50f..2f69b20c6b911 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedSignificantTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedSignificantTerms.java @@ -39,7 +39,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedTerms.java index 067744c6d5d88..33d9c6cfaf4b9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/ParsedTerms.java @@ -37,7 +37,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java index 84525d888d516..dc616ca7512be 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java @@ -33,8 +33,8 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantLongTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantLongTerms.java index 4dd41fc0194b9..46e8cea7abc36 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantLongTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantLongTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantStringTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantStringTerms.java index 9468d12268e78..d8d93ad7ae159 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantStringTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantStringTerms.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java index 7d43d8372f6e9..00d5cb0a83c55 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java @@ -33,8 +33,8 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java index f2e4dc1e1bbcf..7e2c20c14dafb 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringRareTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringRareTerms.java index 30e74d193b877..574e2409b71d0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringRareTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringRareTerms.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.SetBackedScalingCuckooFilter; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java index 6c9eced36c921..c985bf770d4a7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/StringTerms.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java index 32ae11306f750..fd944374eeaaa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java @@ -33,8 +33,8 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java index 99519641c3669..9e2aa85bb1dd8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.bucket.terms; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedRareTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedRareTerms.java index 1e35e0cf6d107..e330174ef18f3 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedRareTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedRareTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.SetBackedScalingCuckooFilter; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedSignificantTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedSignificantTerms.java index 9c5924d96bb56..9384f9e793d81 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedSignificantTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedSignificantTerms.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java index a27074438926b..01902f9449bae 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnmappedTerms.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java index ed623694fa166..db05ac84b4aec 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/UnsignedLongTerms.java @@ -8,8 +8,8 @@ package org.opensearch.search.aggregations.bucket.terms; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.BucketOrder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java index f773310de6e2b..833c7328e14f4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/GND.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/GND.java index d94786a4f8b54..347013498b61f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/GND.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/GND.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/JLHScore.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/JLHScore.java index 6519866a1e03f..fb5d39cbe0e58 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/JLHScore.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/JLHScore.java @@ -34,8 +34,8 @@ import java.io.IOException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java index 794fe3a99af30..86caa6d3b5059 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java index a30c63e5954cb..a2e9262482880 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java index ef52f3bba2754..52078e916ce07 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ScriptHeuristic.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ScriptHeuristic.java index d186366448ff3..33a09548edc83 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ScriptHeuristic.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/ScriptHeuristic.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristic.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristic.java index a7e5db8542f61..8b78c1c4ecb3b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristic.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristic.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.terms.heuristic; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java index 37cdd2227fc8a..f64db066f5a6d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java index 403cd46df1f15..1e36ed0bd9be9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.metrics; import org.HdrHistogram.DoubleHistogram; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java index 0ae23f38c8d0e..398d0054403ac 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java index 37724df8cf1fc..f3ea15addd1cd 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java @@ -34,9 +34,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregationBuilder.java index 7829dd387bf4f..d66053d49b327 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregationBuilder.java index ca5eced9dd15a..a7516a6fd6b24 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregationBuilder.java @@ -34,8 +34,8 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java index bf25eb549022e..91cce9137049d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java index 06e6d4c6dd2c1..bbbdfd60905b0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java index 0adee55fc00e5..1bd56be601a89 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java @@ -36,7 +36,7 @@ import org.apache.lucene.util.packed.PackedInts; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitArray; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.ByteUtils; import org.opensearch.common.util.IntArray; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalAvg.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalAvg.java index d46e64ceee356..36c354b57d3b2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalAvg.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalAvg.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalCardinality.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalCardinality.java index 56456eb80c183..ddb7287cd89a6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalCardinality.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalCardinality.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalExtendedStats.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalExtendedStats.java index 17296f5267193..3e4521e973aac 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalExtendedStats.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalExtendedStats.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoCentroid.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoCentroid.java index d7c284dea439e..dd95e2ac55277 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoCentroid.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoCentroid.java @@ -35,8 +35,8 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.opensearch.core.ParseField; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentileRanks.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentileRanks.java index f517fb678e44f..304bd95e03bc9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentileRanks.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentileRanks.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.metrics; import org.HdrHistogram.DoubleHistogram; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentiles.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentiles.java index 6bfab8c26ac9a..b28d207cc192f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentiles.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalHDRPercentiles.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.metrics; import org.HdrHistogram.DoubleHistogram; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMax.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMax.java index fb4b5f530b15b..2abb2106681ff 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMax.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMax.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java index a6b0740c3ae4d..1f9e6b0050420 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMin.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMin.java index 1db05a3780d9c..bda0fda82a0d7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMin.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalMin.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java index 2ef86786d4b13..a5508d34589ed 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.support.AggregationPath; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalScriptedMetric.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalScriptedMetric.java index 1b349ed897255..6150940512406 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalScriptedMetric.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalScriptedMetric.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.script.Script; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalStats.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalStats.java index dbb69697e31f5..d21e44ca8e55c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalStats.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalStats.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalSum.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalSum.java index 70f2e359a81e5..51d50ca4240ca 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalSum.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalSum.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java index a87564c0a4766..3ff813d5ba69c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentiles.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentiles.java index cbcb3f0bbcaff..cc1f63614f9ef 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentiles.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTDigestPercentiles.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTopHits.java index 7f1e567351579..39055e06f9be8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTopHits.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalTopHits.java @@ -37,8 +37,8 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits.Relation; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalValueCount.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalValueCount.java index db6a5e74162fd..ede89b1369ae8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalValueCount.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalValueCount.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalWeightedAvg.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalWeightedAvg.java index dac1e46e71710..57cfbcb475f83 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalWeightedAvg.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalWeightedAvg.java @@ -31,8 +31,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregationBuilder.java index fb7449137bbfa..ae16efb09d3fa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java index 24952b2e71f9c..7e59d0abc5419 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregationBuilder.java index 02598b676e1bb..770d58c966320 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedScriptedMetric.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedScriptedMetric.java index 7325e2d9a8da6..760adf2b7a350 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedScriptedMetric.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedScriptedMetric.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java index 9426166f67f3e..7585844b6bbb9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesAggregationBuilder.java index 98ecd9796d959..38c6e99003828 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesAggregationBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesConfig.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesConfig.java index 92df7fe2a332e..58f2076b2891d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesConfig.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesMethod.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesMethod.java index 0b107fd8f95f0..dfd3905392685 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesMethod.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/PercentilesMethod.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ObjectParser; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java index e160809bd1d53..419837dde788e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java index 2f4d68227ca42..f60f402939462 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java @@ -36,7 +36,7 @@ import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.util.ObjectArray; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregationBuilder.java index 411c66a881fdf..99c55cfc3ceba 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/StatsAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregationBuilder.java index af1f2e93053d4..ec84a22037257 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/TDigestState.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/TDigestState.java index e551d0e9d4fa2..f0a4236c90cd4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/TDigestState.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/TDigestState.java @@ -33,8 +33,8 @@ import com.tdunning.math.stats.AVLTreeDigest; import com.tdunning.math.stats.Centroid; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index 78b1ada7d1f13..0d12a9deb8f55 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregationBuilder.java index ba9f1bcd9d167..0258a2c87cb20 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java index a92bf2231104d..019be14f7ae82 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java index bd4414e0ebece..49e8abd46bf5c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.PipelineAggregationBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java index 3d130dd74bb5b..d712582c8d96b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java index 41ee184602d36..efdf9b46a0c5f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java @@ -33,10 +33,10 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.search.aggregations.AggregationExecutionException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsParser.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsParser.java index dfae96026b870..ba6fb92875057 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsParser.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsParser.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java index fb0c03e83de57..d156e5aa05e73 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.AggregationBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java index 71c87584c48c1..76516bd5d8f98 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java index 289928bacd26b..c9d1e474fafa8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java index c6a2e1ea6cda2..70ac8b326f523 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java index 0ba38bbf92195..5ec5bdffe0ce9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java index 0146e35f37859..b35f7493691a9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java @@ -33,10 +33,10 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Rounding; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/EwmaModel.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/EwmaModel.java index 9dca9cfab2946..a63eba9355a6d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/EwmaModel.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/EwmaModel.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java index 3232bfa4e4544..165253379f146 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltLinearModel.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltLinearModel.java index 743bf677bf038..6ac075c5714d6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltLinearModel.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltLinearModel.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltWintersModel.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltWintersModel.java index 96aafb2fde415..7c48e319b76bf 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltWintersModel.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/HoltWintersModel.java @@ -35,8 +35,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalBucketMetricValue.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalBucketMetricValue.java index 69f0439652e3e..40ecb6830b9f4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalBucketMetricValue.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalBucketMetricValue.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalDerivative.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalDerivative.java index 941be36d8e86c..090543d0bbb9d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalDerivative.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalDerivative.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalExtendedStatsBucket.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalExtendedStatsBucket.java index 26781eba3e0be..5c8a77e9c760a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalExtendedStatsBucket.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalExtendedStatsBucket.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.metrics.InternalExtendedStats; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalPercentilesBucket.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalPercentilesBucket.java index 6ec5ddb09dd5f..753299f49730f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalPercentilesBucket.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalPercentilesBucket.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalSimpleValue.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalSimpleValue.java index 054a96ba8c43e..c6b1ea0e1c1e7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalSimpleValue.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalSimpleValue.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalStatsBucket.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalStatsBucket.java index 2754376059daf..cee433aeaab41 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalStatsBucket.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/InternalStatsBucket.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.metrics.InternalStats; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/LinearModel.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/LinearModel.java index d07f62d186373..9035beecc1d6d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/LinearModel.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/LinearModel.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java index aace39e5ec15f..0b3172c74e1de 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java index c4a41f765bdc6..eb9ffb2e488ae 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgModel.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgModel.java index 79609aa7076b2..ab415943ee676 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgModel.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgModel.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java index 655d5fd760383..1b023bc6efc94 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java @@ -44,9 +44,9 @@ import java.util.Objects; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.ParseFieldRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java index 9027b3476e3a1..1e569b5caa020 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java index 0c7ce717fd849..43dc8c6714663 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java @@ -35,8 +35,8 @@ import com.carrotsearch.hppc.DoubleArrayList; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java index 9f292a4571db1..a3b1441fb058b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimpleModel.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimpleModel.java index f8832720a9574..290f164d5a8db 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimpleModel.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimpleModel.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations.pipeline; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java index d3da799bf01a4..cc394d85b7093 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java index 1c6400457f677..2bf769780dc19 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInfo.java b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInfo.java index 535229079f2da..6801aacf095c6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInfo.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInfo.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.support; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/BaseMultiValuesSourceFieldConfig.java b/server/src/main/java/org/opensearch/search/aggregations/support/BaseMultiValuesSourceFieldConfig.java index 0ea3a20c32cfe..4a924cbd8305d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/BaseMultiValuesSourceFieldConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/BaseMultiValuesSourceFieldConfig.java @@ -11,9 +11,9 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.TriConsumer; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java index 644c6796b3cfb..a15608b85ad22 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java @@ -9,8 +9,8 @@ package org.opensearch.search.aggregations.support; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java index fe484f2d61826..e3f914ca259f6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.support; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java index 4f3d84adb0836..9418823bab1b3 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java @@ -34,8 +34,8 @@ import org.opensearch.core.ParseField; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceParseHelper.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceParseHelper.java index f25e7437ebce3..ce09a4738e7d0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceParseHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceParseHelper.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.support; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.AbstractObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java index 255181fed3f61..59fa2e03f0bc3 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java @@ -33,9 +33,9 @@ package org.opensearch.search.aggregations.support; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java index 479991cb21e1f..70382369d5615 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java @@ -33,8 +33,8 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.AbstractObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchBackpressureStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchBackpressureStats.java index 185b3c3fb73ec..973ff735f82d2 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchBackpressureStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchBackpressureStats.java @@ -10,9 +10,9 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.backpressure.settings.SearchBackpressureMode; diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java index ab50b454f02f4..678c19d83fb96 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java @@ -9,9 +9,9 @@ package org.opensearch.search.backpressure.stats; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.backpressure.trackers.CpuUsageTracker; diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java index e22cbe5c41e33..302350104bd3a 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java @@ -9,9 +9,9 @@ package org.opensearch.search.backpressure.stats; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.backpressure.trackers.CpuUsageTracker; diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java index a684a0f669429..a5f7d7061dbac 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/CpuUsageTracker.java @@ -8,8 +8,8 @@ package org.opensearch.search.backpressure.trackers; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java index c4ae412cc0b10..666cb7d605017 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/ElapsedTimeTracker.java @@ -8,8 +8,8 @@ package org.opensearch.search.backpressure.trackers; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java index 5cc612257207b..fbd2155c0ef89 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java @@ -11,8 +11,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.MovingAverage; diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java index cc1348e94cfb6..8f772c91750ab 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTracker.java @@ -8,7 +8,7 @@ package org.opensearch.search.backpressure.trackers; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.tasks.TaskCancellation; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java b/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java index a0f0c22bdd818..26a7738177759 100644 --- a/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java @@ -34,9 +34,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index f324d0039eb5a..3cb3e7a142e1d 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -37,10 +37,10 @@ import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilderException.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilderException.java index a884e5d1be8b5..83e5cf2c86958 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilderException.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilderException.java @@ -33,7 +33,7 @@ package org.opensearch.search.builder; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java b/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java index dc83a023d5bf7..288ca9339f8bd 100644 --- a/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java +++ b/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java @@ -32,10 +32,10 @@ package org.opensearch.search.collapse; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java b/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java index c4d159187bb8e..0223699b7fd89 100644 --- a/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java +++ b/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java @@ -38,9 +38,9 @@ import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; import org.opensearch.common.collect.HppcMaps; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/dfs/DfsPhaseExecutionException.java b/server/src/main/java/org/opensearch/search/dfs/DfsPhaseExecutionException.java index 2572bc1a73b6a..f875304cae4fd 100644 --- a/server/src/main/java/org/opensearch/search/dfs/DfsPhaseExecutionException.java +++ b/server/src/main/java/org/opensearch/search/dfs/DfsPhaseExecutionException.java @@ -32,7 +32,7 @@ package org.opensearch.search.dfs; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.SearchException; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java index f5166c124a811..b46dff0dbe8ef 100644 --- a/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java +++ b/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java @@ -39,8 +39,8 @@ import org.apache.lucene.search.TermStatistics; import org.apache.lucene.util.BytesRef; import org.opensearch.common.collect.HppcMaps; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java index d82af3e55ee6d..b82764c38f747 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java @@ -48,7 +48,7 @@ import org.opensearch.common.document.DocumentField; import org.opensearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.opensearch.common.lucene.search.Queries; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhaseExecutionException.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhaseExecutionException.java index b1a31525f704e..ff52a57ffe7bb 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchPhaseExecutionException.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhaseExecutionException.java @@ -32,7 +32,7 @@ package org.opensearch.search.fetch; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.SearchException; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index dcc7d60495a5e..a8560863d8d98 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -32,8 +32,8 @@ package org.opensearch.search.fetch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchPhaseResult; diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index 047bf1b2074e0..ce4c59fc77489 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -32,8 +32,8 @@ package org.opensearch.search.fetch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; diff --git a/server/src/main/java/org/opensearch/search/fetch/ScrollQueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/ScrollQueryFetchSearchResult.java index cfba9ae461bb5..415350b4c5dc7 100644 --- a/server/src/main/java/org/opensearch/search/fetch/ScrollQueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/ScrollQueryFetchSearchResult.java @@ -32,8 +32,8 @@ package org.opensearch.search.fetch; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.query.QuerySearchResult; diff --git a/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java index 5c2a7d5bf974e..017e74a12b61c 100644 --- a/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java +++ b/server/src/main/java/org/opensearch/search/fetch/ShardFetchRequest.java @@ -36,8 +36,8 @@ import org.apache.lucene.search.ScoreDoc; import org.opensearch.action.search.SearchShardTask; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.search.RescoreDocIds; import org.opensearch.search.dfs.AggregatedDfs; diff --git a/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java index 32b2f71522856..4e74017217217 100644 --- a/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/fetch/ShardFetchSearchRequest.java @@ -36,8 +36,8 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.OriginalIndices; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.RescoreDocIds; import org.opensearch.search.dfs.AggregatedDfs; diff --git a/server/src/main/java/org/opensearch/search/fetch/StoredFieldsContext.java b/server/src/main/java/org/opensearch/search/fetch/StoredFieldsContext.java index 94df87201cf27..e8c1dc57627fb 100644 --- a/server/src/main/java/org/opensearch/search/fetch/StoredFieldsContext.java +++ b/server/src/main/java/org/opensearch/search/fetch/StoredFieldsContext.java @@ -32,10 +32,10 @@ package org.opensearch.search.fetch; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java index dbb7258c2c855..5b9b9e1e70cfa 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java @@ -34,10 +34,10 @@ import org.opensearch.common.Booleans; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourcePhase.java index 5a42d47662d4c..b94a3e3564373 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourcePhase.java @@ -34,7 +34,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.opensearch.OpenSearchException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.SearchHit; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/opensearch/search/fetch/subphase/FieldAndFormat.java index d7e131084e61e..f7e4b06624c76 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/FieldAndFormat.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/FieldAndFormat.java @@ -34,9 +34,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContent; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java index c666ff883fcee..162c79c28f982 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java @@ -35,11 +35,11 @@ import org.apache.lucene.search.highlight.SimpleFragmenter; import org.apache.lucene.search.highlight.SimpleSpanFragmenter; import org.opensearch.Version; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java index 77f8e82df50f2..c087ad6b71344 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java @@ -47,7 +47,7 @@ import org.apache.lucene.search.vectorhighlight.SingleFragListBuilder; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.util.CollectionUtils; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.TextSearchInfo; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java index 02c05cfdc0513..cc0723ed7a432 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.NamedObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java index 16e20f123e457..7a29a53304dc3 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java @@ -32,11 +32,11 @@ package org.opensearch.search.fetch.subphase.highlight; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -47,7 +47,7 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A field highlighted with its highlighted fragments. diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java index f19039b2c0af3..eb5f4f3c14eb2 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/PlainHighlighter.java @@ -47,7 +47,7 @@ import org.apache.lucene.search.highlight.TextFragment; import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.CollectionUtil; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.search.fetch.FetchContext; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java index ce20fd0b74bb9..d4bc3544cc184 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java @@ -46,7 +46,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CollectionUtil; import org.opensearch.common.CheckedSupplier; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.core.common.Strings; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.IdFieldMapper; diff --git a/server/src/main/java/org/opensearch/search/internal/AliasFilter.java b/server/src/main/java/org/opensearch/search/internal/AliasFilter.java index d50911759aa44..408f67f5002d9 100644 --- a/server/src/main/java/org/opensearch/search/internal/AliasFilter.java +++ b/server/src/main/java/org/opensearch/search/internal/AliasFilter.java @@ -32,9 +32,9 @@ package org.opensearch.search.internal; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; diff --git a/server/src/main/java/org/opensearch/search/internal/InternalScrollSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/InternalScrollSearchRequest.java index b546ba886a3db..6c407e6ba3401 100644 --- a/server/src/main/java/org/opensearch/search/internal/InternalScrollSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/InternalScrollSearchRequest.java @@ -34,8 +34,8 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.action.search.SearchShardTask; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.Scroll; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java b/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java index d59922dcbe8b4..1561d18f3040a 100644 --- a/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java +++ b/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java @@ -33,9 +33,9 @@ package org.opensearch.search.internal; import org.opensearch.action.search.SearchResponseSections; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchContextId.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchContextId.java index 714b5f2a34ba0..3e0d2b1d6c638 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchContextId.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchContextId.java @@ -32,9 +32,9 @@ package org.opensearch.search.internal; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index 3a0cf564422c0..d2f6bc234e752 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -43,21 +43,21 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedFunction; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.Rewriteable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.AliasFilterParsingException; import org.opensearch.indices.InvalidAliasNameException; import org.opensearch.search.Scroll; diff --git a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java index 3d29916f0bdb4..1d4a424550c12 100644 --- a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java @@ -36,7 +36,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.CheckedBiConsumer; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java index 5ada11b59556b..d4292b85b20a5 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java +++ b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java @@ -16,9 +16,9 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.SearchPhaseResult; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java index e2599fd78908c..512e400c54784 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java @@ -11,9 +11,9 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java index 3e473f070d9c6..7d1bac4463117 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineWithMetrics.java @@ -9,8 +9,8 @@ package org.opensearch.search.pipeline; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.metrics.OperationMetrics; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.ingest.ConfigurationUtils; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java b/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java index 0864fecb6c7f1..0206b9b6cf716 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java +++ b/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java @@ -8,9 +8,9 @@ package org.opensearch.search.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineInfo.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineInfo.java index ce38f3bfbac3e..b2075a51ff732 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineInfo.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineInfo.java @@ -9,8 +9,8 @@ package org.opensearch.search.pipeline; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineMetadata.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineMetadata.java index bfbf5cd24bf92..e8842cd67a895 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineMetadata.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineMetadata.java @@ -13,8 +13,8 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineProcessingException.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineProcessingException.java index a8fd0a8e42750..5e175a7240fd8 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineProcessingException.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineProcessingException.java @@ -10,7 +10,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 83a7a0564467e..ae02068fbcc12 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -29,12 +29,12 @@ import org.opensearch.cluster.service.ClusterManagerTaskKeys; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.gateway.GatewayService; diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java index 4261bfe99160a..68aca990f0f1e 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineStats.java @@ -8,11 +8,11 @@ package org.opensearch.search.pipeline; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.common.metrics.OperationStats; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/profile/NetworkTime.java b/server/src/main/java/org/opensearch/search/profile/NetworkTime.java index 17ae53e8a5828..45d8c2883cb4a 100644 --- a/server/src/main/java/org/opensearch/search/profile/NetworkTime.java +++ b/server/src/main/java/org/opensearch/search/profile/NetworkTime.java @@ -9,9 +9,9 @@ package org.opensearch.search.profile; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java index 0ce99cd44e4fc..89c3d7504de66 100644 --- a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java +++ b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java @@ -33,9 +33,9 @@ package org.opensearch.search.profile; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.InstantiatingObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java index 01a0612f14c8a..502d8e4852588 100644 --- a/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java +++ b/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java @@ -32,9 +32,9 @@ package org.opensearch.search.profile; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.profile.aggregation.AggregationProfileShardResult; import org.opensearch.search.profile.query.QueryProfileShardResult; diff --git a/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java b/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java index f584c05b0a45d..53cf1f2739e77 100644 --- a/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java +++ b/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java @@ -32,9 +32,9 @@ package org.opensearch.search.profile; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -52,7 +52,7 @@ import java.util.Map; import java.util.TreeSet; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A container class to hold all the profile results across all shards. Internally diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java index 8e2e00a02893a..046e929821ab5 100644 --- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java @@ -32,9 +32,9 @@ package org.opensearch.search.profile.aggregation; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -45,7 +45,7 @@ import java.util.Collections; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A container class to hold the profile results for a single shard in the request. diff --git a/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java b/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java index 73dd2214959ad..26edc14e6f9f8 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java +++ b/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; @@ -48,7 +48,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Public interface and serialization container for profiled timings of the diff --git a/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java index 7f954372e1342..a9f3d4aaf7885 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java +++ b/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java @@ -32,9 +32,9 @@ package org.opensearch.search.profile.query; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -45,7 +45,7 @@ import java.util.Collections; import java.util.List; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * A container class to hold the profile results for a single shard in the request. diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhaseExecutionException.java b/server/src/main/java/org/opensearch/search/query/QueryPhaseExecutionException.java index 50dd5f04418f2..43bd87183d0be 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryPhaseExecutionException.java +++ b/server/src/main/java/org/opensearch/search/query/QueryPhaseExecutionException.java @@ -32,7 +32,7 @@ package org.opensearch.search.query; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.SearchException; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchRequest.java b/server/src/main/java/org/opensearch/search/query/QuerySearchRequest.java index 650a9930f2db0..776b8fbb9fca2 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchRequest.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.search.dfs.AggregatedDfs; import org.opensearch.search.internal.ShardSearchContextId; diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index a0c2970625472..dcf940a4edcb4 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -35,8 +35,8 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.TotalHits; import org.opensearch.common.io.stream.DelayableWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.search.DocValueFormat; import org.opensearch.search.RescoreDocIds; diff --git a/server/src/main/java/org/opensearch/search/query/ScrollQuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/ScrollQuerySearchResult.java index ef0827bd0b187..0cdc8749253f0 100644 --- a/server/src/main/java/org/opensearch/search/query/ScrollQuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/ScrollQuerySearchResult.java @@ -32,8 +32,8 @@ package org.opensearch.search.query; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/main/java/org/opensearch/search/rescore/QueryRescoreMode.java b/server/src/main/java/org/opensearch/search/rescore/QueryRescoreMode.java index 818ce18cba6d2..17a20f8dab198 100644 --- a/server/src/main/java/org/opensearch/search/rescore/QueryRescoreMode.java +++ b/server/src/main/java/org/opensearch/search/rescore/QueryRescoreMode.java @@ -32,9 +32,9 @@ package org.opensearch.search.rescore; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java b/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java index 0ed8628430d3a..aea10755d4e42 100644 --- a/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java +++ b/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.rescore; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/rescore/RescorerBuilder.java b/server/src/main/java/org/opensearch/search/rescore/RescorerBuilder.java index 0dfaf2ae90d40..d4094298f0c5f 100644 --- a/server/src/main/java/org/opensearch/search/rescore/RescorerBuilder.java +++ b/server/src/main/java/org/opensearch/search/rescore/RescorerBuilder.java @@ -32,11 +32,11 @@ package org.opensearch.search.rescore; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/search/searchafter/SearchAfterBuilder.java b/server/src/main/java/org/opensearch/search/searchafter/SearchAfterBuilder.java index 52f8fc17489ca..516b388ce2186 100644 --- a/server/src/main/java/org/opensearch/search/searchafter/SearchAfterBuilder.java +++ b/server/src/main/java/org/opensearch/search/searchafter/SearchAfterBuilder.java @@ -38,12 +38,12 @@ import org.apache.lucene.search.SortedSetSortField; import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java b/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java index 9ff69a174ee95..31e03f5ef511e 100644 --- a/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java @@ -41,9 +41,9 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/search/sort/BucketedSort.java b/server/src/main/java/org/opensearch/search/sort/BucketedSort.java index 43a98fb5b1b39..cc82b85781097 100644 --- a/server/src/main/java/org/opensearch/search/sort/BucketedSort.java +++ b/server/src/main/java/org/opensearch/search/sort/BucketedSort.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorable; import org.opensearch.common.lucene.ScorerAware; -import org.opensearch.common.util.BigArray; +import org.opensearch.core.common.util.BigArray; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitArray; import org.opensearch.common.util.DoubleArray; diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index 0b7b9cd07c300..eb1c8639d7492 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -41,9 +41,9 @@ import org.apache.lucene.search.SortField; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.time.DateMathParser; import org.opensearch.common.time.DateUtils; diff --git a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java index 7c4f7697a8a0d..76df59617414a 100644 --- a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java @@ -43,12 +43,12 @@ import org.apache.lucene.util.BitSet; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoDistance; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.util.BigArrays; diff --git a/server/src/main/java/org/opensearch/search/sort/MinAndMax.java b/server/src/main/java/org/opensearch/search/sort/MinAndMax.java index 7e655ca029035..16d8f0f6533bd 100644 --- a/server/src/main/java/org/opensearch/search/sort/MinAndMax.java +++ b/server/src/main/java/org/opensearch/search/sort/MinAndMax.java @@ -33,9 +33,9 @@ package org.opensearch.search.sort; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/sort/NestedSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/NestedSortBuilder.java index 0dc84d819a1fc..feabdfce4cd21 100644 --- a/server/src/main/java/org/opensearch/search/sort/NestedSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/NestedSortBuilder.java @@ -33,9 +33,9 @@ package org.opensearch.search.sort; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java index 3b70865f71d14..8bc819b520d2b 100644 --- a/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java @@ -35,8 +35,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java index a7489ac8614ed..6cb847c5fb739 100644 --- a/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java @@ -39,9 +39,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/search/sort/SortBuilder.java b/server/src/main/java/org/opensearch/search/sort/SortBuilder.java index d9db268aa2209..5bffb8a9ca56e 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/SortBuilder.java @@ -36,9 +36,9 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.join.ToChildBlockJoinQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/search/sort/SortMode.java b/server/src/main/java/org/opensearch/search/sort/SortMode.java index eda46b9bb0f0e..028b907ebe864 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortMode.java +++ b/server/src/main/java/org/opensearch/search/sort/SortMode.java @@ -32,9 +32,9 @@ package org.opensearch.search.sort; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/search/sort/SortOrder.java b/server/src/main/java/org/opensearch/search/sort/SortOrder.java index cfb917329ceb3..ed83a0667c5e7 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortOrder.java +++ b/server/src/main/java/org/opensearch/search/sort/SortOrder.java @@ -32,9 +32,9 @@ package org.opensearch.search.sort; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/search/sort/SortValue.java b/server/src/main/java/org/opensearch/search/sort/SortValue.java index 210df3b7d3605..0c46428bc02d0 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortValue.java +++ b/server/src/main/java/org/opensearch/search/sort/SortValue.java @@ -32,10 +32,10 @@ package org.opensearch.search.sort; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/search/suggest/SortBy.java b/server/src/main/java/org/opensearch/search/suggest/SortBy.java index 4d0b058d1ffe1..5b2235b10317f 100644 --- a/server/src/main/java/org/opensearch/search/suggest/SortBy.java +++ b/server/src/main/java/org/opensearch/search/suggest/SortBy.java @@ -32,9 +32,9 @@ package org.opensearch.search.suggest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/search/suggest/Suggest.java b/server/src/main/java/org/opensearch/search/suggest/Suggest.java index 75c6f63bb86a4..9020a036ade0a 100644 --- a/server/src/main/java/org/opensearch/search/suggest/Suggest.java +++ b/server/src/main/java/org/opensearch/search/suggest/Suggest.java @@ -34,19 +34,19 @@ import org.apache.lucene.util.CollectionUtil; import org.opensearch.common.CheckedFunction; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.action.search.RestSearchAction; import org.opensearch.search.aggregations.Aggregation; @@ -65,7 +65,7 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** * Top level suggest result, containing the result for each suggestion. diff --git a/server/src/main/java/org/opensearch/search/suggest/SuggestBuilder.java b/server/src/main/java/org/opensearch/search/suggest/SuggestBuilder.java index 73bf3866975d1..3daa4ac019cd5 100644 --- a/server/src/main/java/org/opensearch/search/suggest/SuggestBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/SuggestBuilder.java @@ -32,11 +32,11 @@ package org.opensearch.search.suggest; import org.opensearch.common.Nullable; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java index 821f3fa33b39a..94dea8a65293a 100644 --- a/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java @@ -35,10 +35,10 @@ import org.apache.lucene.analysis.Analyzer; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java index 607bd73c99b57..2ca642292a7c0 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java @@ -33,9 +33,9 @@ import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java index c897b36f14ff9..c7498635d2553 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java @@ -40,7 +40,7 @@ import org.apache.lucene.search.suggest.document.TopSuggestDocs; import org.apache.lucene.search.suggest.document.TopSuggestDocsCollector; import org.apache.lucene.util.CharsRefBuilder; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.Suggester; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java index d469e01c8aab8..9cd32db4e2a98 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java @@ -35,10 +35,10 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.util.PriorityQueue; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -57,7 +57,7 @@ import java.util.Objects; import java.util.Set; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.search.SearchHit.unknownMetaFieldConsumer; import static org.opensearch.search.suggest.Suggest.COMPARATOR; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java index 314adf6b8e40c..6f3c3f471b47c 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -33,9 +33,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java b/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java index 5b44f97332ff5..22157817a7124 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; import org.apache.lucene.util.automaton.Operations; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/RegexOptions.java b/server/src/main/java/org/opensearch/search/suggest/completion/RegexOptions.java index 3f1e874c1d78f..fb00d6b186314 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/RegexOptions.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/RegexOptions.java @@ -36,9 +36,9 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java index afd46f1a8164c..337227821f688 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java @@ -43,8 +43,8 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/Laplace.java b/server/src/main/java/org/opensearch/search/suggest/phrase/Laplace.java index 0e729f1a4e4d6..1d2d7861a3587 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/Laplace.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/Laplace.java @@ -36,8 +36,8 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java b/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java index 4a32f4469dee1..f1dba9793ba9e 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java @@ -36,9 +36,9 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java index 6c3b490d5bc44..4b1c43bf7a317 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java @@ -42,7 +42,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRefBuilder; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestion.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestion.java index 091dfa83c873f..b342f628aa4ac 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestion.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestion.java @@ -33,9 +33,9 @@ package org.opensearch.search.suggest.phrase; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java index 906d40a15f674..8621eb1704053 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -34,10 +34,10 @@ import org.apache.lucene.analysis.Analyzer; import org.opensearch.OpenSearchParseException; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/SmoothingModel.java b/server/src/main/java/org/opensearch/search/suggest/phrase/SmoothingModel.java index d6dd6a2e4c2b9..154413d7a6d50 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/SmoothingModel.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/SmoothingModel.java @@ -32,8 +32,8 @@ package org.opensearch.search.suggest.phrase; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/StupidBackoff.java b/server/src/main/java/org/opensearch/search/suggest/phrase/StupidBackoff.java index d69141fa61fbb..74a1c447bfe31 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/StupidBackoff.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/StupidBackoff.java @@ -36,8 +36,8 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.util.BytesRef; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; diff --git a/server/src/main/java/org/opensearch/search/suggest/term/TermSuggester.java b/server/src/main/java/org/opensearch/search/suggest/term/TermSuggester.java index 0c9a3bcc2e168..0c18ae429442b 100644 --- a/server/src/main/java/org/opensearch/search/suggest/term/TermSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/term/TermSuggester.java @@ -39,8 +39,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRefBuilder; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.text.Text; import org.opensearch.search.suggest.Suggester; import org.opensearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.opensearch.search.suggest.phrase.DirectCandidateGenerator; diff --git a/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestion.java b/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestion.java index 2f9bbb9d7465a..9924d62555836 100644 --- a/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestion.java +++ b/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestion.java @@ -33,9 +33,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestionBuilder.java index 8819e1304fbcd..5af9116be2cd0 100644 --- a/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestionBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/term/TermSuggestionBuilder.java @@ -39,10 +39,10 @@ import org.apache.lucene.search.spell.NGramDistance; import org.apache.lucene.search.spell.StringDistance; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; diff --git a/server/src/main/java/org/opensearch/snapshots/ConcurrentSnapshotExecutionException.java b/server/src/main/java/org/opensearch/snapshots/ConcurrentSnapshotExecutionException.java index aa2c0ab42e42e..e0b4d3bf49d2e 100644 --- a/server/src/main/java/org/opensearch/snapshots/ConcurrentSnapshotExecutionException.java +++ b/server/src/main/java/org/opensearch/snapshots/ConcurrentSnapshotExecutionException.java @@ -32,8 +32,8 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/InFlightShardSnapshotStates.java b/server/src/main/java/org/opensearch/snapshots/InFlightShardSnapshotStates.java index f31d8c74074a3..30c4f9483baec 100644 --- a/server/src/main/java/org/opensearch/snapshots/InFlightShardSnapshotStates.java +++ b/server/src/main/java/org/opensearch/snapshots/InFlightShardSnapshotStates.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.Nullable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoryShardId; import org.opensearch.repositories.ShardGenerations; diff --git a/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java b/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java index 942a491c3ad90..3c7a14fb2854a 100644 --- a/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java +++ b/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java @@ -50,7 +50,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; diff --git a/server/src/main/java/org/opensearch/snapshots/InvalidSnapshotNameException.java b/server/src/main/java/org/opensearch/snapshots/InvalidSnapshotNameException.java index 5af529c72458c..9dce7127ccf37 100644 --- a/server/src/main/java/org/opensearch/snapshots/InvalidSnapshotNameException.java +++ b/server/src/main/java/org/opensearch/snapshots/InvalidSnapshotNameException.java @@ -32,8 +32,8 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java b/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java index 8762dd5a2a2c9..1ed805c567d99 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java @@ -32,16 +32,16 @@ package org.opensearch.snapshots; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 7df24e5357555..6edf78b54aa70 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -83,11 +83,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/main/java/org/opensearch/snapshots/Snapshot.java b/server/src/main/java/org/opensearch/snapshots/Snapshot.java index d084fad1c2b21..e7c92195eff08 100644 --- a/server/src/main/java/org/opensearch/snapshots/Snapshot.java +++ b/server/src/main/java/org/opensearch/snapshots/Snapshot.java @@ -32,9 +32,9 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotCreationException.java b/server/src/main/java/org/opensearch/snapshots/SnapshotCreationException.java index 184fcd5ceb9f8..a7f16637f6778 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotCreationException.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotCreationException.java @@ -32,7 +32,7 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotException.java b/server/src/main/java/org/opensearch/snapshots/SnapshotException.java index 9c78834907e85..09d213aa2b051 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotException.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotException.java @@ -34,8 +34,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java index 40b574dc6dc66..aec3aebd93585 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java @@ -32,9 +32,9 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInProgressException.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInProgressException.java index cd202ea7e5f98..ae330a1500388 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotInProgressException.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInProgressException.java @@ -33,8 +33,8 @@ package org.opensearch.snapshots; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInUseDeletionException.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInUseDeletionException.java index e93bf5ab0cd91..11080c1c39167 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotInUseDeletionException.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInUseDeletionException.java @@ -8,8 +8,8 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java index 6bdbcfee29a9a..489086cb782fe 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java @@ -32,23 +32,23 @@ package org.opensearch.snapshots; import org.opensearch.Version; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.repositories.IndexId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.time.Instant; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotMissingException.java b/server/src/main/java/org/opensearch/snapshots/SnapshotMissingException.java index 2508ab89108eb..ca8f89897f333 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotMissingException.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotMissingException.java @@ -32,8 +32,8 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotRestoreException.java b/server/src/main/java/org/opensearch/snapshots/SnapshotRestoreException.java index 8afd64571c0ac..c1239ecd7df51 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotRestoreException.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotRestoreException.java @@ -32,7 +32,7 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java index 3e328476841b6..3af508b62b55c 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java @@ -33,18 +33,18 @@ package org.opensearch.snapshots; import org.opensearch.OpenSearchParseException; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; -import org.opensearch.index.snapshots.IndexShardSnapshotFailedException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index 68a464eee191e..0fcd3761c98ba 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -49,7 +49,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.concurrent.GatedCloseable; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexService; @@ -58,8 +58,8 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; -import org.opensearch.index.snapshots.IndexShardSnapshotFailedException; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.IndexShardSnapshotStatus.Stage; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 3946e4363298c..cd9c2c4ce325a 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -83,14 +83,14 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.component.AbstractLifecycleComponent; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusRequest.java b/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusRequest.java index 90120076c967b..7db9fdac510ab 100644 --- a/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusRequest.java +++ b/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusRequest.java @@ -34,10 +34,10 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.SnapshotsInProgress; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusResponse.java b/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusResponse.java index d1df715ba629a..0ae7a8367d1a8 100644 --- a/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusResponse.java +++ b/server/src/main/java/org/opensearch/snapshots/UpdateIndexShardSnapshotStatusResponse.java @@ -32,7 +32,7 @@ package org.opensearch.snapshots; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/package-info.java b/server/src/main/java/org/opensearch/snapshots/package-info.java index f43509cf671f9..a573d7a136620 100644 --- a/server/src/main/java/org/opensearch/snapshots/package-info.java +++ b/server/src/main/java/org/opensearch/snapshots/package-info.java @@ -45,7 +45,7 @@ *

    *
  1. First the {@link org.opensearch.snapshots.SnapshotsService} determines the primary shards' assignments for all indices that are * being snapshotted and creates a {@code SnapshotsInProgress.Entry} with state {@code STARTED} and adds the map of - * {@link org.opensearch.index.shard.ShardId} to {@link org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus} that tracks + * {@link org.opensearch.core.index.shard.ShardId} to {@link org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus} that tracks * the assignment of which node is to snapshot which shard. All shard snapshots are executed on the shard's primary node. Thus all shards * for which the primary node was found to have a healthy copy of the shard are marked as being in state {@code INIT} in this map. If the * primary for a shard is unassigned, it is marked as {@code MISSING} in this map. In case the primary is initializing at this point, it is diff --git a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java index eee411d8c7131..28a09fd80b408 100644 --- a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java +++ b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java @@ -33,9 +33,9 @@ package org.opensearch.tasks; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java b/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java index d78a4480700da..e6ce092d7516e 100644 --- a/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java +++ b/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java @@ -8,9 +8,9 @@ package org.opensearch.tasks; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/tasks/Task.java b/server/src/main/java/org/opensearch/tasks/Task.java index 8fba67dec4778..3fa62ebbdcbed 100644 --- a/server/src/main/java/org/opensearch/tasks/Task.java +++ b/server/src/main/java/org/opensearch/tasks/Task.java @@ -38,7 +38,7 @@ import org.opensearch.action.ActionResponse; import org.opensearch.action.NotifyOnceListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java index ebce26c4bbfbc..7ff27fa1096dc 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationService.java @@ -41,8 +41,8 @@ import org.opensearch.action.support.ChannelActionListener; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.EmptyTransportResponseHandler; import org.opensearch.transport.TransportChannel; diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java index 2ccb3738b1235..cca898fdd844f 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java @@ -8,9 +8,9 @@ package org.opensearch.tasks; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancelledException.java b/server/src/main/java/org/opensearch/tasks/TaskCancelledException.java index 5f43837c02e7f..38e50f56b24b2 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancelledException.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancelledException.java @@ -32,7 +32,7 @@ package org.opensearch.tasks; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/tasks/TaskId.java b/server/src/main/java/org/opensearch/tasks/TaskId.java index e1f198dba14ad..e7420fce397e2 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskId.java +++ b/server/src/main/java/org/opensearch/tasks/TaskId.java @@ -34,9 +34,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/tasks/TaskInfo.java b/server/src/main/java/org/opensearch/tasks/TaskInfo.java index d66647d134d43..2f259f62b6909 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskInfo.java +++ b/server/src/main/java/org/opensearch/tasks/TaskInfo.java @@ -34,10 +34,10 @@ import org.opensearch.Version; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.ObjectParserHelper; diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceStats.java b/server/src/main/java/org/opensearch/tasks/TaskResourceStats.java index 94eb9478403fd..59526cd229001 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceStats.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceStats.java @@ -10,9 +10,9 @@ import org.opensearch.Version; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceUsage.java b/server/src/main/java/org/opensearch/tasks/TaskResourceUsage.java index 2d936895ec793..490adc3e7c637 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceUsage.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceUsage.java @@ -9,9 +9,9 @@ package org.opensearch.tasks; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/tasks/TaskResult.java b/server/src/main/java/org/opensearch/tasks/TaskResult.java index 2387de173c924..b6006636da2e9 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResult.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResult.java @@ -36,10 +36,10 @@ import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.InstantiatingObjectParser; import org.opensearch.common.xcontent.ObjectParserHelper; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/main/java/org/opensearch/tasks/TaskThreadUsage.java b/server/src/main/java/org/opensearch/tasks/TaskThreadUsage.java index 9c78e82c389f3..d30a3d723a15a 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskThreadUsage.java +++ b/server/src/main/java/org/opensearch/tasks/TaskThreadUsage.java @@ -9,9 +9,9 @@ package org.opensearch.tasks; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index d9f73a9b41658..99bb7f95a85e5 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -37,9 +37,9 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.SizeValue; diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java b/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java index 0afcc2ae23d33..5087624ec0b3f 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java @@ -32,8 +32,8 @@ package org.opensearch.threadpool; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java index dec9673d20c40..b4d7e4a3fbf7a 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java @@ -32,9 +32,9 @@ package org.opensearch.threadpool; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/transport/ActionNotFoundTransportException.java b/server/src/main/java/org/opensearch/transport/ActionNotFoundTransportException.java index 0f12c63a1b58e..11c1e99a065ac 100644 --- a/server/src/main/java/org/opensearch/transport/ActionNotFoundTransportException.java +++ b/server/src/main/java/org/opensearch/transport/ActionNotFoundTransportException.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/ActionTransportException.java b/server/src/main/java/org/opensearch/transport/ActionTransportException.java index c9fbddec941b3..97e9a986db7f4 100644 --- a/server/src/main/java/org/opensearch/transport/ActionTransportException.java +++ b/server/src/main/java/org/opensearch/transport/ActionTransportException.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.transport.TransportAddress; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/BindTransportException.java b/server/src/main/java/org/opensearch/transport/BindTransportException.java index 918906e2b4759..7ef01d2367f11 100644 --- a/server/src/main/java/org/opensearch/transport/BindTransportException.java +++ b/server/src/main/java/org/opensearch/transport/BindTransportException.java @@ -32,7 +32,7 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/BytesTransportRequest.java b/server/src/main/java/org/opensearch/transport/BytesTransportRequest.java index cb623adf1c26b..3fd84432639d4 100644 --- a/server/src/main/java/org/opensearch/transport/BytesTransportRequest.java +++ b/server/src/main/java/org/opensearch/transport/BytesTransportRequest.java @@ -33,9 +33,9 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java index 58cbc406612f4..1deddf93b6252 100644 --- a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java +++ b/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java @@ -33,11 +33,11 @@ package org.opensearch.transport; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.io.Streams; -import org.opensearch.common.io.stream.BytesStream; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.io.OutputStream; diff --git a/server/src/main/java/org/opensearch/transport/ConnectTransportException.java b/server/src/main/java/org/opensearch/transport/ConnectTransportException.java index 0267cab686264..257cb442ffd99 100644 --- a/server/src/main/java/org/opensearch/transport/ConnectTransportException.java +++ b/server/src/main/java/org/opensearch/transport/ConnectTransportException.java @@ -33,8 +33,8 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java index c4fd78fd72ef4..3b97a81faf192 100644 --- a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java @@ -32,7 +32,7 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; /** diff --git a/server/src/main/java/org/opensearch/transport/Header.java b/server/src/main/java/org/opensearch/transport/Header.java index f73d3fcc74e72..ba4a8cfef5534 100644 --- a/server/src/main/java/org/opensearch/transport/Header.java +++ b/server/src/main/java/org/opensearch/transport/Header.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.concurrent.ThreadContext; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/InboundAggregator.java b/server/src/main/java/org/opensearch/transport/InboundAggregator.java index c8cfaa8339e33..60d45c8b3ad2b 100644 --- a/server/src/main/java/org/opensearch/transport/InboundAggregator.java +++ b/server/src/main/java/org/opensearch/transport/InboundAggregator.java @@ -34,9 +34,9 @@ import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/transport/InboundDecoder.java b/server/src/main/java/org/opensearch/transport/InboundDecoder.java index 96d8967be1841..0dd39ed73d787 100644 --- a/server/src/main/java/org/opensearch/transport/InboundDecoder.java +++ b/server/src/main/java/org/opensearch/transport/InboundDecoder.java @@ -33,9 +33,9 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/transport/InboundHandler.java b/server/src/main/java/org/opensearch/transport/InboundHandler.java index 37604151603d0..bb04f149d39a9 100644 --- a/server/src/main/java/org/opensearch/transport/InboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/InboundHandler.java @@ -37,10 +37,10 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; -import org.opensearch.common.io.stream.ByteBufferStreamInput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; diff --git a/server/src/main/java/org/opensearch/transport/InboundMessage.java b/server/src/main/java/org/opensearch/transport/InboundMessage.java index 31262ab22a0ea..e19e5b45025aa 100644 --- a/server/src/main/java/org/opensearch/transport/InboundMessage.java +++ b/server/src/main/java/org/opensearch/transport/InboundMessage.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.common.bytes.ReleasableBytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index be269dd229f22..4a4f928860819 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/transport/NetworkMessage.java b/server/src/main/java/org/opensearch/transport/NetworkMessage.java index 6b53935a863dd..a24dd282031fd 100644 --- a/server/src/main/java/org/opensearch/transport/NetworkMessage.java +++ b/server/src/main/java/org/opensearch/transport/NetworkMessage.java @@ -32,7 +32,7 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ThreadContext; /** diff --git a/server/src/main/java/org/opensearch/transport/NoSeedNodeLeftException.java b/server/src/main/java/org/opensearch/transport/NoSeedNodeLeftException.java index 4a75889d08e17..66365b4c529cb 100644 --- a/server/src/main/java/org/opensearch/transport/NoSeedNodeLeftException.java +++ b/server/src/main/java/org/opensearch/transport/NoSeedNodeLeftException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/NoSuchRemoteClusterException.java b/server/src/main/java/org/opensearch/transport/NoSuchRemoteClusterException.java index cc6c349191b8f..ee3b11172cb30 100644 --- a/server/src/main/java/org/opensearch/transport/NoSuchRemoteClusterException.java +++ b/server/src/main/java/org/opensearch/transport/NoSuchRemoteClusterException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/NodeDisconnectedException.java b/server/src/main/java/org/opensearch/transport/NodeDisconnectedException.java index e12f84a5de70c..93379e6023b44 100644 --- a/server/src/main/java/org/opensearch/transport/NodeDisconnectedException.java +++ b/server/src/main/java/org/opensearch/transport/NodeDisconnectedException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/NodeNotConnectedException.java b/server/src/main/java/org/opensearch/transport/NodeNotConnectedException.java index a94adb0ec5678..915633273bd6e 100644 --- a/server/src/main/java/org/opensearch/transport/NodeNotConnectedException.java +++ b/server/src/main/java/org/opensearch/transport/NodeNotConnectedException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/NodeShouldNotConnectException.java b/server/src/main/java/org/opensearch/transport/NodeShouldNotConnectException.java index 7f6521716411b..24fb8e1269d56 100644 --- a/server/src/main/java/org/opensearch/transport/NodeShouldNotConnectException.java +++ b/server/src/main/java/org/opensearch/transport/NodeShouldNotConnectException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/NotSerializableTransportException.java b/server/src/main/java/org/opensearch/transport/NotSerializableTransportException.java index 6cec3da191a49..487dc113743f8 100644 --- a/server/src/main/java/org/opensearch/transport/NotSerializableTransportException.java +++ b/server/src/main/java/org/opensearch/transport/NotSerializableTransportException.java @@ -32,7 +32,7 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index 4c54b4bd8b6ff..3e493267242fb 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -40,7 +40,7 @@ import org.opensearch.action.NotifyOnceListener; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedSupplier; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.network.CloseableChannel; import org.opensearch.common.transport.NetworkExceptionHelper; diff --git a/server/src/main/java/org/opensearch/transport/OutboundMessage.java b/server/src/main/java/org/opensearch/transport/OutboundMessage.java index 25dfe7acc27e8..d969351425acd 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundMessage.java +++ b/server/src/main/java/org/opensearch/transport/OutboundMessage.java @@ -32,12 +32,12 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ThreadContext; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java index f09cbc1c89eee..53d71c135a8dd 100644 --- a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java +++ b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchTimeoutException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.concurrent.BaseFuture; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java index 7684082368b2c..b1eefb9fac245 100644 --- a/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java @@ -38,9 +38,9 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/main/java/org/opensearch/transport/ReceiveTimeoutTransportException.java b/server/src/main/java/org/opensearch/transport/ReceiveTimeoutTransportException.java index a8985d2175024..a2d48373cd43d 100644 --- a/server/src/main/java/org/opensearch/transport/ReceiveTimeoutTransportException.java +++ b/server/src/main/java/org/opensearch/transport/ReceiveTimeoutTransportException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java index 7d6fe07666096..d95c0c647916b 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java @@ -38,7 +38,7 @@ import org.opensearch.action.support.ContextPreservingActionListener; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java index bc5a33d1b77ed..9271fd36b8cbe 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java +++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java @@ -32,9 +32,9 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java index 902b8f8367336..76afeb20d18cf 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java @@ -39,7 +39,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ContextPreservingActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/transport/RemoteTransportException.java b/server/src/main/java/org/opensearch/transport/RemoteTransportException.java index a02f470c3a4ae..041a70795b8de 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteTransportException.java +++ b/server/src/main/java/org/opensearch/transport/RemoteTransportException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.OpenSearchWrapperException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.transport.TransportAddress; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java b/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java index b292e6b75ebdd..bd1ccda2cbe30 100644 --- a/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java +++ b/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/transport/ResponseHandlerFailureTransportException.java b/server/src/main/java/org/opensearch/transport/ResponseHandlerFailureTransportException.java index 9de4ad04d6ed0..8eb31b3593b1b 100644 --- a/server/src/main/java/org/opensearch/transport/ResponseHandlerFailureTransportException.java +++ b/server/src/main/java/org/opensearch/transport/ResponseHandlerFailureTransportException.java @@ -32,7 +32,7 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/SendRequestTransportException.java b/server/src/main/java/org/opensearch/transport/SendRequestTransportException.java index 929292a2cea58..a70c8d3361ba2 100644 --- a/server/src/main/java/org/opensearch/transport/SendRequestTransportException.java +++ b/server/src/main/java/org/opensearch/transport/SendRequestTransportException.java @@ -34,7 +34,7 @@ import org.opensearch.OpenSearchWrapperException; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java index 981df456e9394..5e00704b3baaf 100644 --- a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java @@ -45,9 +45,9 @@ import org.opensearch.common.Booleans; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/main/java/org/opensearch/transport/TcpChannel.java b/server/src/main/java/org/opensearch/transport/TcpChannel.java index 14012ec5dd43c..2e13fb49dcd15 100644 --- a/server/src/main/java/org/opensearch/transport/TcpChannel.java +++ b/server/src/main/java/org/opensearch/transport/TcpChannel.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.network.CloseableChannel; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/transport/TcpHeader.java b/server/src/main/java/org/opensearch/transport/TcpHeader.java index e82488420c6d1..78353a9a80403 100644 --- a/server/src/main/java/org/opensearch/transport/TcpHeader.java +++ b/server/src/main/java/org/opensearch/transport/TcpHeader.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 07bf9162d9355..0ea383d6d48d7 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -43,12 +43,12 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Booleans; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.network.CloseableChannel; import org.opensearch.common.network.NetworkAddress; @@ -69,7 +69,7 @@ import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.node.Node; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java index 6ab4a86071746..8ad6010800ad8 100644 --- a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java @@ -32,9 +32,9 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/transport/TransportDecompressor.java b/server/src/main/java/org/opensearch/transport/TransportDecompressor.java index 6ecb1dba7ad16..3e53c10c6a069 100644 --- a/server/src/main/java/org/opensearch/transport/TransportDecompressor.java +++ b/server/src/main/java/org/opensearch/transport/TransportDecompressor.java @@ -34,8 +34,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.compress.Compressor; import org.opensearch.common.compress.CompressorFactory; diff --git a/server/src/main/java/org/opensearch/transport/TransportException.java b/server/src/main/java/org/opensearch/transport/TransportException.java index 38d1d5d76c644..1522f2378bca3 100644 --- a/server/src/main/java/org/opensearch/transport/TransportException.java +++ b/server/src/main/java/org/opensearch/transport/TransportException.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java index 8491ef96f7f08..7b64b328469ad 100644 --- a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java @@ -34,10 +34,10 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.unit.TimeValue; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/transport/TransportInfo.java b/server/src/main/java/org/opensearch/transport/TransportInfo.java index e2911fb982098..0ee8672e8df8c 100644 --- a/server/src/main/java/org/opensearch/transport/TransportInfo.java +++ b/server/src/main/java/org/opensearch/transport/TransportInfo.java @@ -33,8 +33,8 @@ package org.opensearch.transport; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.transport.BoundTransportAddress; diff --git a/server/src/main/java/org/opensearch/transport/TransportInterceptor.java b/server/src/main/java/org/opensearch/transport/TransportInterceptor.java index 755f307f64bb8..f4b003cae4864 100644 --- a/server/src/main/java/org/opensearch/transport/TransportInterceptor.java +++ b/server/src/main/java/org/opensearch/transport/TransportInterceptor.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; /** * This interface allows plugins to intercept requests on both the sender and the receiver side. diff --git a/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java b/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java index b90355cf5e775..e48267d943739 100644 --- a/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java +++ b/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.ActionListener; import org.opensearch.common.AsyncBiFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.component.Lifecycle; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.metrics.CounterMetric; diff --git a/server/src/main/java/org/opensearch/transport/TransportLogger.java b/server/src/main/java/org/opensearch/transport/TransportLogger.java index 1f7facbca49ec..d7d00a20964f8 100644 --- a/server/src/main/java/org/opensearch/transport/TransportLogger.java +++ b/server/src/main/java/org/opensearch/transport/TransportLogger.java @@ -34,10 +34,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/main/java/org/opensearch/transport/TransportMessage.java b/server/src/main/java/org/opensearch/transport/TransportMessage.java index f40d8526df818..78216047d530e 100644 --- a/server/src/main/java/org/opensearch/transport/TransportMessage.java +++ b/server/src/main/java/org/opensearch/transport/TransportMessage.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.transport.TransportAddress; /** diff --git a/server/src/main/java/org/opensearch/transport/TransportRequest.java b/server/src/main/java/org/opensearch/transport/TransportRequest.java index b089ed538e041..95b038303f530 100644 --- a/server/src/main/java/org/opensearch/transport/TransportRequest.java +++ b/server/src/main/java/org/opensearch/transport/TransportRequest.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.tasks.TaskAwareRequest; import org.opensearch.tasks.TaskId; diff --git a/server/src/main/java/org/opensearch/transport/TransportResponse.java b/server/src/main/java/org/opensearch/transport/TransportResponse.java index 59cbcc8c25dc0..73713fa1447a8 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponse.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponse.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index 40bac6fb8440e..674a62fb75b7d 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.function.Function; diff --git a/server/src/main/java/org/opensearch/transport/TransportSerializationException.java b/server/src/main/java/org/opensearch/transport/TransportSerializationException.java index 5ce651a5030c8..1a5f9cec28ea1 100644 --- a/server/src/main/java/org/opensearch/transport/TransportSerializationException.java +++ b/server/src/main/java/org/opensearch/transport/TransportSerializationException.java @@ -32,7 +32,7 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index 021bf39fc5a1f..c2163ae1d490a 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -44,10 +44,10 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; import org.opensearch.common.component.AbstractLifecycleComponent; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Streamables; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.Loggers; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.ClusterSettings; diff --git a/server/src/main/java/org/opensearch/transport/TransportStats.java b/server/src/main/java/org/opensearch/transport/TransportStats.java index 20175f6b5b279..29544754a5729 100644 --- a/server/src/main/java/org/opensearch/transport/TransportStats.java +++ b/server/src/main/java/org/opensearch/transport/TransportStats.java @@ -32,9 +32,9 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/BuildTests.java b/server/src/test/java/org/opensearch/BuildTests.java index a8ac431d672e8..a7daa5d901109 100644 --- a/server/src/test/java/org/opensearch/BuildTests.java +++ b/server/src/test/java/org/opensearch/BuildTests.java @@ -33,9 +33,9 @@ package org.opensearch; import org.opensearch.core.util.FileSystemUtils; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 8b0d6faeb1e61..2656810af0379 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -61,7 +61,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.routing.UnsupportedWeightedRoutingStateException; import org.opensearch.cluster.service.ClusterManagerThrottlingException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; import org.opensearch.common.breaker.CircuitBreaker; @@ -69,18 +69,20 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NotSerializableExceptionWrapper; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.CancellableThreadsTests; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.index.snapshots.IndexShardSnapshotException; +import org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.discovery.MasterNotDiscoveredException; import org.opensearch.env.ShardLockObtainFailedException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.RecoveryEngineException; import org.opensearch.index.query.QueryShardException; import org.opensearch.index.seqno.RetentionLeaseAlreadyExistsException; @@ -89,7 +91,7 @@ import org.opensearch.index.shard.IllegalIndexShardStateException; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.PrimaryShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.indices.IndexTemplateMissingException; import org.opensearch.indices.InvalidIndexTemplateException; @@ -99,7 +101,7 @@ import org.opensearch.ingest.IngestProcessorException; import org.opensearch.cluster.coordination.NodeHealthCheckFailureException; import org.opensearch.repositories.RepositoryException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; import org.opensearch.search.SearchContextMissingException; import org.opensearch.search.SearchException; @@ -720,7 +722,7 @@ public void testThatIdsAreUnique() { public void testIds() { Map> ids = new HashMap<>(); - ids.put(0, org.opensearch.index.snapshots.IndexShardSnapshotFailedException.class); + ids.put(0, IndexShardSnapshotFailedException.class); ids.put(1, org.opensearch.search.dfs.DfsPhaseExecutionException.class); ids.put(2, org.opensearch.common.util.CancellableThreads.ExecutionCancelledException.class); ids.put(3, org.opensearch.discovery.ClusterManagerNotDiscoveredException.class); @@ -760,7 +762,7 @@ public void testIds() { ids.put(37, org.opensearch.index.mapper.MapperException.class); ids.put(38, org.opensearch.indices.InvalidTypeNameException.class); ids.put(39, org.opensearch.snapshots.SnapshotRestoreException.class); - ids.put(40, org.opensearch.common.ParsingException.class); + ids.put(40, ParsingException.class); ids.put(41, org.opensearch.index.shard.IndexShardClosedException.class); ids.put(42, org.opensearch.indices.recovery.RecoverFilesRecoveryException.class); ids.put(43, org.opensearch.index.translog.TruncatedTranslogException.class); @@ -782,7 +784,7 @@ public void testIds() { ids.put(59, null); // was OpenSearchRejectedExecutionException, which is no longer an instance of OpenSearchException ids.put(60, null); // EarlyTerminationException was removed in 6.0 ids.put(61, null); // RoutingValidationException was removed in 5.0 - ids.put(62, org.opensearch.common.io.stream.NotSerializableExceptionWrapper.class); + ids.put(62, NotSerializableExceptionWrapper.class); ids.put(63, org.opensearch.indices.AliasFilterParsingException.class); ids.put(64, null); // DeleteByQueryFailedEngineException was removed in 3.0 ids.put(65, org.opensearch.gateway.GatewayException.class); @@ -815,7 +817,7 @@ public void testIds() { ids.put(95, null); ids.put(96, org.opensearch.snapshots.InvalidSnapshotNameException.class); ids.put(97, org.opensearch.index.shard.IllegalIndexShardStateException.class); - ids.put(98, org.opensearch.index.snapshots.IndexShardSnapshotException.class); + ids.put(98, IndexShardSnapshotException.class); ids.put(99, org.opensearch.index.shard.IndexShardNotStartedException.class); ids.put(100, org.opensearch.action.search.SearchPhaseExecutionException.class); ids.put(101, org.opensearch.transport.ActionNotFoundTransportException.class); diff --git a/server/src/test/java/org/opensearch/ExceptionsHelperTests.java b/server/src/test/java/org/opensearch/ExceptionsHelperTests.java index 4c65eadf46e89..f5513271b8479 100644 --- a/server/src/test/java/org/opensearch/ExceptionsHelperTests.java +++ b/server/src/test/java/org/opensearch/ExceptionsHelperTests.java @@ -36,15 +36,15 @@ import org.apache.commons.codec.DecoderException; import org.apache.lucene.index.CorruptIndexException; import org.opensearch.action.OriginalIndices; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.QueryShardException; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.RemoteClusterAware; diff --git a/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java b/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java index a0bd8202abbbe..ca94462160f23 100644 --- a/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java +++ b/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java @@ -43,11 +43,11 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.coordination.NoClusterManagerBlockService; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; @@ -58,14 +58,14 @@ import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryShardException; import org.opensearch.index.shard.IndexShardRecoveringException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.NodeClosedException; import org.opensearch.repositories.RepositoryException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.ScriptException; import org.opensearch.search.SearchContextMissingException; import org.opensearch.search.SearchParseException; diff --git a/server/src/test/java/org/opensearch/action/DocWriteResponseTests.java b/server/src/test/java/org/opensearch/action/DocWriteResponseTests.java index d867881c76c20..46b4eae77f99b 100644 --- a/server/src/test/java/org/opensearch/action/DocWriteResponseTests.java +++ b/server/src/test/java/org/opensearch/action/DocWriteResponseTests.java @@ -34,14 +34,14 @@ import org.opensearch.action.DocWriteResponse.Result; import org.opensearch.action.support.replication.ReplicationResponse.ShardInfo; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java b/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java index 17e424ee81e7e..c06791f0f0245 100644 --- a/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java +++ b/server/src/test/java/org/opensearch/action/DynamicActionRegistryTests.java @@ -12,8 +12,8 @@ import org.opensearch.action.main.MainAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.TransportAction; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.extensions.action.ExtensionAction; import org.opensearch.extensions.action.ExtensionTransportAction; import org.opensearch.rest.NamedRoute; diff --git a/server/src/test/java/org/opensearch/action/OriginalIndicesTests.java b/server/src/test/java/org/opensearch/action/OriginalIndicesTests.java index ef2eb88c91b2f..41752f6b64bd3 100644 --- a/server/src/test/java/org/opensearch/action/OriginalIndicesTests.java +++ b/server/src/test/java/org/opensearch/action/OriginalIndicesTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java index eae56629d1eac..23b1a8d8462cb 100644 --- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java +++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java @@ -12,7 +12,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; diff --git a/server/src/test/java/org/opensearch/action/ShardOperationFailedExceptionTests.java b/server/src/test/java/org/opensearch/action/ShardOperationFailedExceptionTests.java index 6de0704e66217..8232a78a0aba0 100644 --- a/server/src/test/java/org/opensearch/action/ShardOperationFailedExceptionTests.java +++ b/server/src/test/java/org/opensearch/action/ShardOperationFailedExceptionTests.java @@ -33,9 +33,10 @@ package org.opensearch.action; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.action.ShardOperationFailedException; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/ShardValidateQueryRequestTests.java b/server/src/test/java/org/opensearch/action/ShardValidateQueryRequestTests.java index 726f77b0ddf13..e24dc7f99cad9 100644 --- a/server/src/test/java/org/opensearch/action/ShardValidateQueryRequestTests.java +++ b/server/src/test/java/org/opensearch/action/ShardValidateQueryRequestTests.java @@ -34,12 +34,12 @@ import org.opensearch.action.admin.indices.validate.query.ShardValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesModule; import org.opensearch.search.SearchModule; import org.opensearch.search.internal.AliasFilter; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java index 8479c6835cd25..abc3e9c303e33 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java @@ -45,12 +45,12 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import static java.util.Collections.emptyMap; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 56c9f70dc5575..59180539243f5 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -49,7 +49,7 @@ import org.opensearch.cluster.node.DiscoveryNodes.Builder; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java index 37ead5d588f84..4136d1cf31f3a 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.node.DiscoveryNodes.Builder; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/decommission/awareness/DeleteDecommissionStateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/decommission/awareness/DeleteDecommissionStateRequestTests.java index 1a95b77cc1024..f44d43597f50f 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/decommission/awareness/DeleteDecommissionStateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/decommission/awareness/DeleteDecommissionStateRequestTests.java @@ -10,7 +10,7 @@ import org.opensearch.action.admin.cluster.decommission.awareness.delete.DeleteDecommissionStateRequest; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestTests.java index 1dbefd17b200b..66356d4916bf1 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.Priority; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.Strings; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponsesTests.java index c83048593415d..cb199c664eb5a 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponsesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponsesTests.java @@ -43,8 +43,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.DeprecationHandler; @@ -52,7 +52,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.AbstractSerializingTestCase; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthActionTests.java index 0b7fe4c8b1dfc..e62dc9b400e13 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthActionTests.java @@ -44,8 +44,8 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.Randomness; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestTests.java index 4d7982b063071..412b546e134b7 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.node.info; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.util.HashSet; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index 1e2e085333e50..cbf7032b50ca5 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.metrics.OperationStats; import org.opensearch.discovery.DiscoveryStats; import org.opensearch.cluster.coordination.PendingClusterStateStats; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java index f61c3c7676b54..130c60539b74b 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.util.HashSet; import java.util.Set; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java index e7026e9bc34cb..c16584877c07d 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksTests.java @@ -45,8 +45,8 @@ import org.opensearch.action.support.replication.ClusterStateCreationUtils; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java index 3fddf4b481b45..96f2365412e7b 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java @@ -20,8 +20,8 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.tasks.CancellableTask; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index 843fd85d81197..abbaf3a3f7b96 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -46,10 +46,10 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskTests.java index abbd1c645b7c9..84a7fdff3dfa2 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.node.tasks; import org.opensearch.action.search.SearchAction; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TestTaskPlugin.java index d501f2aec3c5c..b46d4fcfea2c9 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TestTaskPlugin.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TestTaskPlugin.java @@ -53,10 +53,10 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index c071b1622dd60..6885278b08413 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -51,10 +51,10 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java index 284e6f538adb9..81d7074977253 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java @@ -8,8 +8,8 @@ package org.opensearch.action.admin.cluster.remotestore.restore; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestTests.java index 8f0a6bba791f6..8a32f36e8fd10 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestTests.java @@ -9,7 +9,7 @@ package org.opensearch.action.admin.cluster.remotestore.stats; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java index 7d8e1ad5c7016..a476b66719d3f 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java @@ -8,13 +8,13 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.opensearch.action.support.DefaultShardOperationFailedException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index 096dfc9145d57..747dc692b1d5d 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -9,7 +9,7 @@ package org.opensearch.action.admin.cluster.remotestore.stats; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.Map; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 94d2eae31c040..fc057b71b15f8 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -8,14 +8,14 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index 086c5331fbc80..f3f6a570c31ba 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -25,7 +25,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestTests.java index 203bfd0db284a..3e9759411733e 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestTests.java @@ -31,7 +31,7 @@ package org.opensearch.action.admin.cluster.repositories.put; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index b057b583caeb8..6ba7961ccecf3 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -40,11 +40,11 @@ import org.opensearch.cluster.routing.allocation.command.AllocationCommand; import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.NetworkModule; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteTests.java index 859d8ce3bb734..5e535c77e80a7 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteTests.java @@ -46,11 +46,11 @@ import org.opensearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java index fad5155d0fd61..6df0563ee1b4e 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.settings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java index ee6f980ebd9ce..6f01f45fc8108 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.settings; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java index 59ece9c1618c3..b2cf673b84152 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java index 4207dca9e3750..a09a580de1475 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java @@ -38,14 +38,14 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.core.common.Strings; import org.opensearch.index.query.RandomQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchModule; import org.opensearch.search.internal.AliasFilter; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestTests.java index 9d0ed8e03d7f2..3b50b0ff87428 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestTests.java @@ -10,7 +10,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.cluster.routing.WeightedRouting; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 6083e74190e1b..7c9d913951d36 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.IndicesOptions.Option; import org.opensearch.action.support.IndicesOptions.WildcardStates; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent.MapParams; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java index 7a294094a21d8..249c87f2eebad 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.snapshots.create; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotInfoTests; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index 02b8f47f0e1de..61c0173b8dec3 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -33,9 +33,9 @@ package org.opensearch.action.admin.cluster.snapshots.get; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotInfoTests; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index bb55ac810ed09..5c20b3b262730 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.snapshots.restore; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatusTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatusTests.java index 8662ed6ed7ae6..c6b429dee3b10 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatusTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatusTests.java @@ -37,9 +37,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.AbstractXContentTestCase; public class SnapshotIndexShardStatusTests extends AbstractXContentTestCase { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatusTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatusTests.java index f58b677f29933..4b81bbf8b2710 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatusTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatusTests.java @@ -38,7 +38,7 @@ import java.util.function.Predicate; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.test.AbstractXContentTestCase; public class SnapshotIndexStatusTests extends AbstractXContentTestCase { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java index bf89204764ed4..a7d1e501cd8b4 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java @@ -35,7 +35,7 @@ import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.UUIDs; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; import org.opensearch.test.AbstractXContentTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestTests.java index 672e5ace8b5ae..e1b8598ef9ccf 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateResponseTests.java index 1cf5ecd9fde9e..01f58bdbc73ff 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/state/ClusterStateResponseTests.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; public class ClusterStateResponseTests extends AbstractWireSerializingTestCase { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/AnalysisStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/AnalysisStatsTests.java index 001836698057c..347e277e042ee 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/AnalysisStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/AnalysisStatsTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.stats; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/MappingStatsTests.java index 9a4b89d975bc8..1b419d7ce77bd 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/MappingStatsTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.stats; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java index 01e2b7fec371c..ac0777b9632db 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java @@ -31,7 +31,7 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java index c73eb16bfa26d..0dea4511304c1 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.ScriptLanguagesInfo; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java index 9a248bec61220..ce1781edbce64 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java index 43209b3de627c..67358fade4b17 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.script.Script; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java index de5cad93980eb..2eb6fa504baf2 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -32,10 +32,10 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java index c33d3a66db296..f478b3e9f5c98 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.ScriptContextInfo; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java index 38f2cbe094809..579ef9ebb1a70 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.ScriptContextInfo.ScriptMethodInfo; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java index f7f8be328deb3..f6fe58ca8b0da 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.ScriptContextInfo.ScriptMethodInfo.ParameterInfo; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java index 0073c0ca79339..265f02304ce00 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java @@ -34,9 +34,9 @@ import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestTests.java index 3112da68fc32e..d4b7075d280ba 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.alias; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponseTests.java index 78c5e6dddcde6..25bb1f8a92687 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.AliasMetadata.Builder; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestTests.java index d710801c489a5..9cb0e9b4329b0 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeResponseTests.java index 00545898ad3e4..769b48450168c 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.analyze; import org.opensearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java index 14a76acca976c..001e0380c8842 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.cache.clear; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractBroadcastResponseTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java index 57a41f08f73f2..030ee6649bba5 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java @@ -22,14 +22,14 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheFactory; import org.opensearch.index.store.remote.filecache.FileCacheTests; import org.opensearch.indices.IndicesService; import org.opensearch.node.Node; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportService; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexRequestTests.java index 355c9b5341049..15e04effe91c6 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexRequestTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.tasks.TaskId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexResponseTests.java index 2de25e8631095..8949dd65d540f 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/close/CloseIndexResponseTests.java @@ -36,13 +36,13 @@ import org.opensearch.action.NoShardAvailableActionException; import org.opensearch.action.admin.indices.close.CloseIndexResponse.IndexResult; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.AbstractWireSerializingTestCase; import org.opensearch.transport.ActionNotFoundTransportException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java index f617585894766..55e4efd8b10df 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java @@ -63,7 +63,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.TaskId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java index 159f05ba6c44d..e4114a622602c 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexResponseTests.java index df391fd316307..d036a65bcd198 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.create; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java index 0d7f451a0f3fc..4df2ef286edf9 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java @@ -33,7 +33,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.admin.indices.datastream.CreateDataStreamAction.Request; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import static org.hamcrest.Matchers.containsString; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java index a88868173b951..5e4e2346a6e5b 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.datastream; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java index 008046aa83dfe..5c7dc04cc6753 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java @@ -42,10 +42,10 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.MetadataDeleteIndexService; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInProgressException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java index 54e83bc764cad..a8db27b1025eb 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.metadata.DataStream; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java index 5863ee7d328bd..da38639650a61 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.datastream.GetDataStreamAction.Response; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.DataStreamTests; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; import java.util.List; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/flush/FlushResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/flush/FlushResponseTests.java index 96d9419161d6f..391d367da0c6d 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/flush/FlushResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/flush/FlushResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.flush; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractBroadcastResponseTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java index 3064847b54332..f6bcec5e8fb42 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.forcemerge; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractBroadcastResponseTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java index 7cad706ef3887..821fb60c82a25 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.forcemerge; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java index 81e73c4889d3b..395cf375127a8 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java @@ -44,7 +44,7 @@ import org.opensearch.common.settings.SettingsFilter; import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.test.transport.CapturingTransport; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java index 401d2bbdf9e89..3719c09719081 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponseTests; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.index.RandomCreateIndexGenerator; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index 5ea5575c3cfc4..4c86b18201f1f 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -34,10 +34,10 @@ import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java index 99a4315b3acee..fa8eeb7389b43 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.mapping.get; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.mapper.MapperService; import org.opensearch.test.AbstractWireSerializingTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 6846ebbb314f8..8ab4598e88af8 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; @@ -49,7 +49,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.RandomCreateIndexGenerator; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexRequestTests.java index 8686101f9d845..a02aa2932c24a 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexRequestTests.java @@ -11,7 +11,7 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexResponseTests.java index 4bd7adc7b48b9..0c0b30515f429 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/open/OpenIndexResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.open; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/refresh/RefreshResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/refresh/RefreshResponseTests.java index da9d9a0cff224..42d4a3d990da1 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/refresh/RefreshResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/refresh/RefreshResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.refresh; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractBroadcastResponseTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexRequestTests.java index 22586514d2c73..a287e474cfc4a 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexRequestTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; public class ResolveIndexRequestTests extends AbstractWireSerializingTestCase { diff --git a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexResponseTests.java index c792ed0e0439f..3e3042b58a8d5 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexResponseTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream; import org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex; import org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 8521673743d23..89cdb84b252e3 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -69,7 +69,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DateFieldMapper; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java index 176c3163697c1..4b23a68f8b383 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -34,12 +34,12 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java index 9f2ea33db939d..233e6166d1caf 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.rollover; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 4596ee4ef77af..65d7c0fc62629 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -74,7 +74,7 @@ import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.shard.DocsStats; import org.opensearch.index.shard.IndexingStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.StoreStats; import org.opensearch.index.warmer.WarmerStats; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java index dfbc1dccb9d6a..0d063c5969ff5 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java @@ -44,7 +44,7 @@ import org.opensearch.common.settings.SettingsFilter; import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestTests.java index 0484f710cda34..f56bab4e5caf7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestTests.java @@ -32,9 +32,9 @@ package org.opensearch.action.admin.indices.settings.get; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponseTests.java index 9eb2f28405b69..154003e036958 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.settings.get; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestSerializationTests.java b/server/src/test/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestSerializationTests.java index 1072c183e164a..ae632d22a3310 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestSerializationTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestSerializationTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.settings.put; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java index 2d353340aa5b0..d29c9cb30a09c 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java @@ -36,7 +36,7 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java index 4742157b3209b..3d21af584ea04 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequestTests; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeResponseTests.java index d8a87d9bffe80..73520f7329a00 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.shrink; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponseTests.java index 9203ec97b7a7f..4b1006a10af79 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponseTests.java @@ -38,8 +38,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsTests.java b/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsTests.java index a3a4be2b93148..8d16082e77533 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/stats/IndicesStatsTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionFuture; import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java index 01655d2ca3e7f..47a5600eef53a 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.template.delete; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java index e499e84ad9df7..ec175055f679b 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.metadata.ComponentTemplate; import org.opensearch.cluster.metadata.ComponentTemplateTests; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java index 39ab3be7c6dbd..a07296945f910 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.template.get; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java index 6826d18938e61..9d170330536c2 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.ComposableIndexTemplateTests; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java index 7f62861d4f332..8b41eeb90b9e4 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index eaf676d58f45d..077b8360edf47 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.ComposableIndexTemplateTests; import org.opensearch.cluster.metadata.Template; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index 87ee57a7a3b9b..5664a1909cceb 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.ComposableIndexTemplateTests; import org.opensearch.cluster.metadata.Template; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java index d73ba0c7e9105..348015958a1e5 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.ComposableIndexTemplateTests; import org.opensearch.cluster.metadata.Template; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/validate/query/QueryExplanationTests.java b/server/src/test/java/org/opensearch/action/admin/indices/validate/query/QueryExplanationTests.java index 370a5782394f1..f30b5f1211e23 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/validate/query/QueryExplanationTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/validate/query/QueryExplanationTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.validate.query; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java index a3894e17b753b..554acd4f92ad0 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.validate.query; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkItemResponseTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkItemResponseTests.java index 3e5a7d5ad5242..708672193665b 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkItemResponseTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkItemResponseTests.java @@ -41,7 +41,7 @@ import org.opensearch.action.index.IndexResponseTests; import org.opensearch.action.update.UpdateResponse; import org.opensearch.action.update.UpdateResponseTests; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkPrimaryExecutionContextTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkPrimaryExecutionContextTests.java index 5159135a22618..4330dd68990dc 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkPrimaryExecutionContextTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkPrimaryExecutionContextTests.java @@ -42,7 +42,7 @@ import org.opensearch.action.update.UpdateRequest; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java index 3a13afac5d977..40d0df61cbb9f 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java @@ -37,7 +37,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.action.index.IndexRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestModifierTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestModifierTests.java index acc612183eac7..e68d7d7d0d447 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestModifierTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestModifierTests.java @@ -37,7 +37,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java index c51c9f7ea77dc..e0e9ea73d9291 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.bulk; import org.opensearch.action.index.IndexRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java index 1be82e045ec4c..c674be8dbba38 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java @@ -39,9 +39,9 @@ import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Requests; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkResponseTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkResponseTests.java index aca2419e2d325..7c758fb25ce85 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkResponseTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkResponseTests.java @@ -39,7 +39,7 @@ import org.opensearch.action.delete.DeleteResponseTests; import org.opensearch.action.index.IndexResponseTests; import org.opensearch.action.update.UpdateResponseTests; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkShardRequestTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkShardRequestTests.java index 5aa4553133190..05c946e73f94d 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkShardRequestTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkShardRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.bulk; import org.opensearch.action.support.WriteRequest.RefreshPolicy; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import static org.apache.lucene.tests.util.TestUtil.randomSimpleString; diff --git a/server/src/test/java/org/opensearch/action/bulk/RetryTests.java b/server/src/test/java/org/opensearch/action/bulk/RetryTests.java index 9da2d482ef5bd..ea02b964b1c58 100644 --- a/server/src/test/java/org/opensearch/action/bulk/RetryTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/RetryTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.update.UpdateRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpClient; import org.junit.After; diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java index cc7b5cb8dc845..c0c35e8c22f4d 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java @@ -65,7 +65,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.IndexingPressureService; @@ -81,12 +81,12 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.opensearch.threadpool.ThreadPool.Names; diff --git a/server/src/test/java/org/opensearch/action/delete/DeleteResponseTests.java b/server/src/test/java/org/opensearch/action/delete/DeleteResponseTests.java index 8c703d62f901c..b5af0ad4c47c9 100644 --- a/server/src/test/java/org/opensearch/action/delete/DeleteResponseTests.java +++ b/server/src/test/java/org/opensearch/action/delete/DeleteResponseTests.java @@ -34,13 +34,13 @@ import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.RandomObjects; diff --git a/server/src/test/java/org/opensearch/action/explain/ExplainRequestTests.java b/server/src/test/java/org/opensearch/action/explain/ExplainRequestTests.java index 6642f5c10522e..3d5f779aada0e 100644 --- a/server/src/test/java/org/opensearch/action/explain/ExplainRequestTests.java +++ b/server/src/test/java/org/opensearch/action/explain/ExplainRequestTests.java @@ -33,9 +33,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesModule; diff --git a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java index cada565748de8..866e13c7faf0e 100644 --- a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java +++ b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java @@ -33,10 +33,10 @@ package org.opensearch.action.explain; import org.apache.lucene.search.Explanation; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestTests.java index 71ce15dcda568..576e5a68dfca6 100644 --- a/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestTests.java +++ b/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.util.ArrayUtils; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 222b348a3ed5e..bfd463f442da3 100644 --- a/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.fieldcaps; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesTests.java index f82bd431f4710..1f31d93997243 100644 --- a/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/opensearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -32,7 +32,7 @@ package org.opensearch.action.fieldcaps; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 253252d76bbc9..382425ce4e57c 100644 --- a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -32,8 +32,8 @@ package org.opensearch.action.fieldcaps; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/get/GetResponseTests.java b/server/src/test/java/org/opensearch/action/get/GetResponseTests.java index 319af46a9d5b7..00ae69c93f5d1 100644 --- a/server/src/test/java/org/opensearch/action/get/GetResponseTests.java +++ b/server/src/test/java/org/opensearch/action/get/GetResponseTests.java @@ -32,10 +32,10 @@ package org.opensearch.action.get; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java index 8aa053b7517fd..5498d685dbb6c 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java @@ -32,8 +32,8 @@ package org.opensearch.action.get; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java index 549d2cf58d32b..71ada43bef116 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java @@ -31,7 +31,7 @@ package org.opensearch.action.get; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetShardRequestTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetShardRequestTests.java index 5bf4f50c50c63..c21d3a25e7f58 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetShardRequestTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetShardRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.get; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.VersionType; import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java index 1184b05461025..9e467aff710df 100644 --- a/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java +++ b/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java @@ -47,15 +47,15 @@ import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; diff --git a/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java b/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java index 21305957d802b..bda9a49bf797b 100644 --- a/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java @@ -36,15 +36,15 @@ import org.opensearch.action.DocWriteRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.replication.ReplicationResponse; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/index/IndexResponseTests.java b/server/src/test/java/org/opensearch/action/index/IndexResponseTests.java index 6e9e06956046e..e52d6d600395e 100644 --- a/server/src/test/java/org/opensearch/action/index/IndexResponseTests.java +++ b/server/src/test/java/org/opensearch/action/index/IndexResponseTests.java @@ -35,13 +35,13 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.RandomObjects; diff --git a/server/src/test/java/org/opensearch/action/ingest/GetPipelineResponseTests.java b/server/src/test/java/org/opensearch/action/ingest/GetPipelineResponseTests.java index 68bb523d594c0..4abd59def0f33 100644 --- a/server/src/test/java/org/opensearch/action/ingest/GetPipelineResponseTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/GetPipelineResponseTests.java @@ -32,8 +32,8 @@ package org.opensearch.action.ingest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java index 8a9eda29a1d7f..a86257fd741c0 100644 --- a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java @@ -32,10 +32,10 @@ package org.opensearch.action.ingest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulateDocumentBaseResultTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulateDocumentBaseResultTests.java index edc0022e0c64a..15fbbf6e833bd 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulateDocumentBaseResultTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulateDocumentBaseResultTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.ingest.IngestDocument; import org.opensearch.test.AbstractXContentTestCase; diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestTests.java index d33f37c5a4e40..35cbc83661c8e 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestTests.java @@ -32,9 +32,9 @@ package org.opensearch.action.ingest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineResponseTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineResponseTests.java index 65e9b2b03d1d5..0ef3cb1133537 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineResponseTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractXContentTestCase; diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulateProcessorResultTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulateProcessorResultTests.java index 4f1c49a340f9f..5b85e345d55be 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulateProcessorResultTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulateProcessorResultTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.ingest.IngestDocument; import org.opensearch.test.AbstractXContentTestCase; diff --git a/server/src/test/java/org/opensearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/opensearch/action/ingest/WriteableIngestDocumentTests.java index da3f9687ae5cf..84f71b48eaf21 100644 --- a/server/src/test/java/org/opensearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/WriteableIngestDocumentTests.java @@ -32,9 +32,9 @@ package org.opensearch.action.ingest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/action/main/MainActionTests.java b/server/src/test/java/org/opensearch/action/main/MainActionTests.java index 479e36c2e13ce..b219e83524e7f 100644 --- a/server/src/test/java/org/opensearch/action/main/MainActionTests.java +++ b/server/src/test/java/org/opensearch/action/main/MainActionTests.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.Transport; diff --git a/server/src/test/java/org/opensearch/action/main/MainResponseTests.java b/server/src/test/java/org/opensearch/action/main/MainResponseTests.java index 27b6a32624070..83b52bb74a56b 100644 --- a/server/src/test/java/org/opensearch/action/main/MainResponseTests.java +++ b/server/src/test/java/org/opensearch/action/main/MainResponseTests.java @@ -36,7 +36,7 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java b/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java index a078966e3aa80..654dbb203b38a 100644 --- a/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java +++ b/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java @@ -33,9 +33,9 @@ package org.opensearch.action.resync; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java index e0c436a943b93..61dd2ac8c14ae 100644 --- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java @@ -48,19 +48,19 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; diff --git a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java index ad2657517df9a..d906c7be15a15 100644 --- a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java @@ -42,9 +42,9 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.MatchAllQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 2e3ff166a6a53..0561f81f96ce6 100644 --- a/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/test/java/org/opensearch/action/search/ClearScrollControllerTests.java b/server/src/test/java/org/opensearch/action/search/ClearScrollControllerTests.java index aeb54cb8c3c0c..e3c7d4741d3ae 100644 --- a/server/src/test/java/org/opensearch/action/search/ClearScrollControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/ClearScrollControllerTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.UUIDs; import org.opensearch.common.util.concurrent.AtomicArray; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; diff --git a/server/src/test/java/org/opensearch/action/search/ClearScrollRequestTests.java b/server/src/test/java/org/opensearch/action/search/ClearScrollRequestTests.java index 6a5885190df39..11c96f6377c13 100644 --- a/server/src/test/java/org/opensearch/action/search/ClearScrollRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/ClearScrollRequestTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.search; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/action/search/CountedCollectorTests.java b/server/src/test/java/org/opensearch/action/search/CountedCollectorTests.java index 7da98dbb9c9b2..ee84788a634f1 100644 --- a/server/src/test/java/org/opensearch/action/search/CountedCollectorTests.java +++ b/server/src/test/java/org/opensearch/action/search/CountedCollectorTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.common.UUIDs; import org.opensearch.common.util.concurrent.AtomicArray; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.dfs.DfsSearchResult; diff --git a/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java b/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java index c03c27f7d7e4d..2c585d152c5d4 100644 --- a/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.IdsQueryBuilder; diff --git a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java index d746e5639fcb1..db371eb9467ba 100644 --- a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java index 6c77e5f6efd5d..24a018d05a16a 100644 --- a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.breaker.NoopCircuitBreaker; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/server/src/test/java/org/opensearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/MultiSearchRequestTests.java index 5795d182e622e..f9d163f251176 100644 --- a/server/src/test/java/org/opensearch/action/search/MultiSearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/MultiSearchRequestTests.java @@ -36,11 +36,11 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.CheckedBiConsumer; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java b/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java index 83bcdef312b39..07a0d3a1d97f2 100644 --- a/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java +++ b/server/src/test/java/org/opensearch/action/search/PitTestsUtil.java @@ -23,7 +23,7 @@ import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.AliasFilter; diff --git a/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java b/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java index 4df269a34c94e..27d04cba204fb 100644 --- a/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java +++ b/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java @@ -42,7 +42,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java index cf838682aa717..9ee3b11f05785 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java @@ -41,10 +41,10 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.AliasFilter; diff --git a/server/src/test/java/org/opensearch/action/search/SearchContextIdTests.java b/server/src/test/java/org/opensearch/action/search/SearchContextIdTests.java index 24f3ca31a1992..65e419936760f 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchContextIdTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchContextIdTests.java @@ -33,13 +33,13 @@ package org.opensearch.action.search; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.index.query.IdsQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.internal.AliasFilter; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java index a007fe0fb44f8..93f9f69e68de0 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java @@ -49,17 +49,17 @@ import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.breaker.NoopCircuitBreaker; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java index 169f2d47743d1..0ea7e503529e8 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java @@ -35,9 +35,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.OriginalIndices; import org.opensearch.action.TimestampParsingException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; @@ -45,9 +45,9 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.shard.IndexShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.InvalidIndexTemplateException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java index 4e351e1424cd0..1b67d805fe814 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -47,7 +47,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/test/java/org/opensearch/action/search/SearchResponseMergerTests.java b/server/src/test/java/org/opensearch/action/search/SearchResponseMergerTests.java index 3f60432140425..1004965c0d50e 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchResponseMergerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchResponseMergerTests.java @@ -38,9 +38,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.search.TransportSearchAction.SearchTimeProvider; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.text.Text; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.text.Text; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java b/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java index c0c0e2d8d9d3a..d6983d111bcde 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java @@ -36,8 +36,8 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchScrollAsyncActionTests.java index b25d6815f0f1d..7ae96e51df76a 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchScrollAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchScrollAsyncActionTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AtomicArray; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.Scroll; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.InternalScrollSearchRequest; diff --git a/server/src/test/java/org/opensearch/action/search/SearchScrollRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchScrollRequestTests.java index ecf356dcd5f82..9d59266242c90 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchScrollRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchScrollRequestTests.java @@ -34,9 +34,9 @@ import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/search/SearchShardIteratorTests.java b/server/src/test/java/org/opensearch/action/search/SearchShardIteratorTests.java index e1a75b3af9f28..489893dde0c0e 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchShardIteratorTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchShardIteratorTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.OriginalIndicesTests; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.routing.GroupShardsIteratorTests; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; diff --git a/server/src/test/java/org/opensearch/action/search/SearchShardTests.java b/server/src/test/java/org/opensearch/action/search/SearchShardTests.java index a50d5685ef5d5..770340240296e 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchShardTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchShardTests.java @@ -32,8 +32,8 @@ package org.opensearch.action.search; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; diff --git a/server/src/test/java/org/opensearch/action/search/ShardSearchFailureTests.java b/server/src/test/java/org/opensearch/action/search/ShardSearchFailureTests.java index 97c2496d4239d..4fa555f0edddc 100644 --- a/server/src/test/java/org/opensearch/action/search/ShardSearchFailureTests.java +++ b/server/src/test/java/org/opensearch/action/search/ShardSearchFailureTests.java @@ -34,13 +34,13 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java index d6de562d616fa..3d76e711788db 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.index.query.IdsQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; diff --git a/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java index 51d9a06c9ac43..413fff98de44c 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java @@ -56,13 +56,13 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.InnerHitBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermsQueryBuilder; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.Scroll; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/server/src/test/java/org/opensearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/opensearch/action/search/TransportSearchHelperTests.java index 160282f784f08..1668f2043fefd 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportSearchHelperTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportSearchHelperTests.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.util.concurrent.AtomicArray; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; diff --git a/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java b/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java index a9e4b34fe5ede..302f3c1efc394 100644 --- a/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java +++ b/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java index 37084fada85ae..0e25091d845b5 100644 --- a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java +++ b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java @@ -35,9 +35,9 @@ import org.opensearch.Version; import org.opensearch.action.support.IndicesOptions.Option; import org.opensearch.action.support.IndicesOptions.WildcardStates; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent.MapParams; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/action/support/RefreshPolicyTests.java b/server/src/test/java/org/opensearch/action/support/RefreshPolicyTests.java index c8e1225b09cb8..32e909a8b8338 100644 --- a/server/src/test/java/org/opensearch/action/support/RefreshPolicyTests.java +++ b/server/src/test/java/org/opensearch/action/support/RefreshPolicyTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.support; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/support/TransportActionFilterChainTests.java b/server/src/test/java/org/opensearch/action/support/TransportActionFilterChainTests.java index fbc7976822501..35c346957ebd2 100644 --- a/server/src/test/java/org/opensearch/action/support/TransportActionFilterChainTests.java +++ b/server/src/test/java/org/opensearch/action/support/TransportActionFilterChainTests.java @@ -38,7 +38,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionResponse; import org.opensearch.action.LatchedActionListener; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.node.Node; import org.opensearch.tasks.Task; diff --git a/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index bf43823f47079..23654c02f0901 100644 --- a/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -36,7 +36,7 @@ import org.opensearch.Version; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.broadcast.BroadcastRequest; import org.opensearch.action.support.broadcast.BroadcastResponse; @@ -60,13 +60,13 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java index 6bf854182d5d6..68b7d97fa1427 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java @@ -33,9 +33,9 @@ import org.opensearch.Version; import org.opensearch.action.support.master.ShardsAcknowledgedResponse; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java index be91efcc4ec32..bcc4861db0a8d 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java @@ -54,14 +54,14 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterManagerThrottlingException; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; import org.opensearch.node.NodeClosedException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java index 66502efd054f0..28f01d0e6ea4a 100644 --- a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java @@ -43,9 +43,9 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java index f653f98840a56..330d43e791555 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.admin.indices.flush.TransportFlushAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ActionTestUtils; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.broadcast.BroadcastRequest; import org.opensearch.action.support.broadcast.BroadcastResponse; @@ -49,18 +49,18 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/support/replication/PendingReplicationActionsTests.java b/server/src/test/java/org/opensearch/action/support/replication/PendingReplicationActionsTests.java index 66d3b843529ab..a26198a5602cc 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/PendingReplicationActionsTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/PendingReplicationActionsTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.index.shard.PrimaryShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java index 652f82409679b..78081a8f83ce3 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java @@ -59,7 +59,7 @@ import org.opensearch.index.shard.IndexShardNotStartedException; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.NodeClosedException; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java index 7f5a1925043dc..5e32193415bd8 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java @@ -37,13 +37,13 @@ import org.opensearch.common.Strings; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.RandomObjects; diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java index d0168a4b863da..089bcf77afbae 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java @@ -70,28 +70,28 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.cluster.ClusterStateChanges; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.test.transport.MockTransportService; diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index 58ce94e6bd581..f7c26ed8fea03 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -54,7 +54,7 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.lease.Releasable; @@ -62,9 +62,9 @@ import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportException; diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java index 0956fddc36d01..c978031103ff2 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java @@ -18,18 +18,18 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.index.stats.IndexingPressurePerShardStats; diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java index a252991d7e119..7a0d0f3814100 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java @@ -51,15 +51,15 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.PrimaryShardClosedException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; diff --git a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java index b12ecb86c8e76..02f245e94ffdc 100644 --- a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java @@ -49,14 +49,14 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/action/termvectors/TermVectorsUnitTests.java b/server/src/test/java/org/opensearch/action/termvectors/TermVectorsUnitTests.java index d762f7c37aa15..c7f14f7a22805 100644 --- a/server/src/test/java/org/opensearch/action/termvectors/TermVectorsUnitTests.java +++ b/server/src/test/java/org/opensearch/action/termvectors/TermVectorsUnitTests.java @@ -51,14 +51,14 @@ import org.apache.lucene.store.Directory; import org.opensearch.LegacyESVersion; import org.opensearch.action.termvectors.TermVectorsRequest.Flag; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.rest.action.document.RestTermVectorsAction; import org.opensearch.tasks.TaskId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java b/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java index b62050a1b8050..cc4abc5343959 100644 --- a/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java +++ b/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java @@ -48,15 +48,15 @@ import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; diff --git a/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java index d94ca9f1e3068..e0ee034f53821 100644 --- a/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/update/UpdateRequestTests.java @@ -37,10 +37,10 @@ import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.replication.ReplicationRequest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -52,7 +52,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.env.Environment; import org.opensearch.index.get.GetResult; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptEngine; diff --git a/server/src/test/java/org/opensearch/action/update/UpdateResponseTests.java b/server/src/test/java/org/opensearch/action/update/UpdateResponseTests.java index 1d1c09ceedefb..9a215bfffb00c 100644 --- a/server/src/test/java/org/opensearch/action/update/UpdateResponseTests.java +++ b/server/src/test/java/org/opensearch/action/update/UpdateResponseTests.java @@ -36,8 +36,8 @@ import org.opensearch.action.index.IndexResponseTests; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.core.xcontent.ToXContent; @@ -46,7 +46,7 @@ import org.opensearch.index.get.GetResult; import org.opensearch.index.get.GetResultTests; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.RandomObjects; diff --git a/server/src/test/java/org/opensearch/client/node/NodeClientHeadersTests.java b/server/src/test/java/org/opensearch/client/node/NodeClientHeadersTests.java index 1f63dba4457a9..36196d0d236c7 100644 --- a/server/src/test/java/org/opensearch/client/node/NodeClientHeadersTests.java +++ b/server/src/test/java/org/opensearch/client/node/NodeClientHeadersTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.support.TransportAction; import org.opensearch.client.AbstractClientHeadersTestCase; import org.opensearch.client.Client; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; diff --git a/server/src/test/java/org/opensearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/opensearch/cluster/ClusterChangedEventTests.java index 9a5866a21dab6..d20a13f90b14f 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterChangedEventTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterChangedEventTests.java @@ -44,7 +44,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.gateway.GatewayService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.TestCustomMetadata; diff --git a/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java index e1294da1e57bc..4ec7db2f3d552 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java @@ -35,7 +35,7 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/ClusterStateTests.java b/server/src/test/java/org/opensearch/cluster/ClusterStateTests.java index 42a291ea5a257..23c2506bf6143 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterStateTests.java @@ -55,9 +55,9 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.TestCustomMetadata; diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java index e5833ea619774..8ba965b3df1ab 100644 --- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java @@ -41,8 +41,8 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingHelper; import org.opensearch.cluster.routing.UnassignedInfo; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.StoreStats; import org.opensearch.monitor.fs.FsInfo; diff --git a/server/src/test/java/org/opensearch/cluster/FeatureAwareTests.java b/server/src/test/java/org/opensearch/cluster/FeatureAwareTests.java index 7d49c6800ad65..da0f81d9b8a5b 100644 --- a/server/src/test/java/org/opensearch/cluster/FeatureAwareTests.java +++ b/server/src/test/java/org/opensearch/cluster/FeatureAwareTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.ClusterState.FeatureAware; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/cluster/action/index/MappingUpdatedActionTests.java b/server/src/test/java/org/opensearch/cluster/action/index/MappingUpdatedActionTests.java index 6d4cc67cd8b45..c3ea1608f1275 100644 --- a/server/src/test/java/org/opensearch/cluster/action/index/MappingUpdatedActionTests.java +++ b/server/src/test/java/org/opensearch/cluster/action/index/MappingUpdatedActionTests.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.Mapping; diff --git a/server/src/test/java/org/opensearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java index 2ee50156cc7cf..564af1841daee 100644 --- a/server/src/test/java/org/opensearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java @@ -58,8 +58,8 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.ArrayList; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java index cb87197a53dea..b423a0f3dc8de 100644 --- a/server/src/test/java/org/opensearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.util.ArrayList; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java index 8ae01e405b392..2688aaa145dc0 100644 --- a/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java @@ -51,11 +51,11 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealthTests.java b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealthTests.java index 6274ecffe9a08..ddd8b635ae84a 100644 --- a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealthTests.java @@ -8,7 +8,7 @@ package org.opensearch.cluster.awarenesshealth; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthSerializationTests.java b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthSerializationTests.java index deb023327ae46..a6cf687f5892b 100644 --- a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthSerializationTests.java +++ b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthSerializationTests.java @@ -8,7 +8,7 @@ package org.opensearch.cluster.awarenesshealth; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java index 8d69480d42d23..59f6edbadcecd 100644 --- a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java @@ -29,7 +29,7 @@ import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java index a02af7556c0a2..04e04bd96a7d3 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java @@ -35,8 +35,8 @@ import org.opensearch.Version; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CompressedStreamUtilsTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CompressedStreamUtilsTests.java index e8faa73315e85..c2843704e0ac5 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CompressedStreamUtilsTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CompressedStreamUtilsTests.java @@ -13,8 +13,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.BytesTransportRequest; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java index 97865d3845c2c..15887d6759458 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java @@ -33,8 +33,8 @@ import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java index 8d3763adfe440..8c9095cd78914 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java index 0862597937e9d..be211a16cdd72 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; import org.opensearch.monitor.NodeHealthService; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index ad9dfd564d648..27146829ad8da 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -41,8 +41,8 @@ import org.opensearch.cluster.NotClusterManagerException; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.monitor.StatusInfo; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java index 6a93f05803484..ac6d885229f9e 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.coordination.LeaderChecker.LeaderCheckRequest; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.monitor.StatusInfo; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/OpenSearchNodeCommandTests.java b/server/src/test/java/org/opensearch/cluster/coordination/OpenSearchNodeCommandTests.java index 853ead71b30fb..47ec3b2c4e9f0 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/OpenSearchNodeCommandTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/OpenSearchNodeCommandTests.java @@ -38,12 +38,12 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.indices.IndicesModule; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java index 947b2273e4d37..97bd683cab9eb 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.monitor.StatusInfo; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 658b8155919d8..4f5bf0f292b5d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.node.Node; diff --git a/server/src/test/java/org/opensearch/cluster/health/ClusterIndexHealthTests.java b/server/src/test/java/org/opensearch/cluster/health/ClusterIndexHealthTests.java index c5f1b58925338..557f7f74db7a7 100644 --- a/server/src/test/java/org/opensearch/cluster/health/ClusterIndexHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/health/ClusterIndexHealthTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTableGenerator; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/health/ClusterShardHealthTests.java b/server/src/test/java/org/opensearch/cluster/health/ClusterShardHealthTests.java index 093820741e5b2..734ad4058630d 100644 --- a/server/src/test/java/org/opensearch/cluster/health/ClusterShardHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/health/ClusterShardHealthTests.java @@ -31,7 +31,7 @@ package org.opensearch.cluster.health; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java index 19458c9f8a4f3..fde0ebe4b4e30 100644 --- a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java @@ -58,7 +58,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/AliasMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/AliasMetadataTests.java index 8220431d408fa..e89eccd2734cb 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/AliasMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/AliasMetadataTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.metadata.AliasMetadata.Builder; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractXContentTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateMetadataTests.java index d04150186aee7..66e00d320dd40 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateMetadataTests.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.metadata; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.test.AbstractNamedWriteableTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateTests.java index 9ec4e9add0e77..9964550d2f8a0 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ComponentTemplateTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractDiffableSerializationTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java index 83ad6c40cc7b2..f6161fb2b62cc 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.metadata; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.test.AbstractNamedWriteableTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateTests.java index fe9384dba4d72..f1888a5a980d4 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractDiffableSerializationTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/DataStreamMetadataTests.java index fcef13d034767..17daae49d5ab5 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/DataStreamMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/DataStreamMetadataTests.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.metadata; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.test.AbstractNamedWriteableTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTemplateTests.java index bb6cac90bca2b..0d2a9a5f5d2fb 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTemplateTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTemplateTests.java @@ -9,7 +9,7 @@ package org.opensearch.cluster.metadata; import org.opensearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTests.java index 7b9c37e19ca24..5b31eb3c2abf7 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/DataStreamTests.java @@ -32,9 +32,9 @@ package org.opensearch.cluster.metadata; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.AbstractSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataSerializationTests.java b/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataSerializationTests.java index 874660dac4e25..e9502a04450f4 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataSerializationTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataSerializationTests.java @@ -13,8 +13,8 @@ import org.opensearch.cluster.decommission.DecommissionAttribute; import org.opensearch.cluster.decommission.DecommissionAttributeMetadata; import org.opensearch.cluster.decommission.DecommissionStatus; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractDiffableSerializationTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataTests.java index 98cecd8439413..d3259dea0d289 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/DecommissionAttributeMetadataTests.java @@ -11,7 +11,7 @@ import org.opensearch.cluster.decommission.DecommissionAttribute; import org.opensearch.cluster.decommission.DecommissionAttributeMetadata; import org.opensearch.cluster.decommission.DecommissionStatus; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.test.AbstractNamedWriteableTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexGraveyardTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexGraveyardTests.java index f6253023fbdd5..ebe5884ac5347 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexGraveyardTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexGraveyardTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentOpenSearchExtension; @@ -43,7 +43,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index 38fb1a657b9e8..ea615c7cfc09a 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -37,11 +37,11 @@ import org.opensearch.action.admin.indices.rollover.MaxSizeCondition; import org.opensearch.action.admin.indices.rollover.RolloverInfo; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; @@ -51,7 +51,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesModule; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java index e79839de7a292..51696ca599d02 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -48,7 +48,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.indices.IndexClosedException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java index d5de57806282d..4648eaa6e3852 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java @@ -31,8 +31,8 @@ package org.opensearch.cluster.metadata; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ManifestTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ManifestTests.java index ce8a7826e605b..534bcd0aed212 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ManifestTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ManifestTests.java @@ -33,12 +33,12 @@ package org.opensearch.cluster.metadata; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index c70fedbaf24bd..da02e6dca680e 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -68,7 +68,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index b3772edcbe27c..5633b874eb423 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index f3c7e73e419db..bf66f577e182b 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceTests.java index 5cffa1931dd39..1863d45af8f2e 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -54,9 +54,9 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceUtils.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceUtils.java index 3f16813dedbb7..6947b6432ccc1 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceUtils.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexStateServiceUtils.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.close.CloseIndexResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlock; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import java.util.Map; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 886d71b159634..4442f192a37ea 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -53,7 +53,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.IndexTemplateMissingException; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataMappingServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataMappingServiceTests.java index 5388c0a866c5f..3574187a2fa01 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataMappingServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataMappingServiceTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.ClusterStateTaskExecutor; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java index 29904aa9e3ff8..f8c706e639445 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java @@ -40,10 +40,10 @@ import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; @@ -51,7 +51,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.plugins.MapperPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java index 1219e684ce5ac..c0f0fc699ba78 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java @@ -46,8 +46,8 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index 0232f671f86de..14f6880419286 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/test/java/org/opensearch/cluster/routing/AllocationIdTests.java b/server/src/test/java/org/opensearch/cluster/routing/AllocationIdTests.java index 0a4ef803068eb..40ce062d6e6e2 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/AllocationIdTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/AllocationIdTests.java @@ -33,11 +33,11 @@ package org.opensearch.cluster.routing; import org.opensearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java index c325517837d09..9000f11ef22bc 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java @@ -21,7 +21,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.settings.Settings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.NodeNotConnectedException; diff --git a/server/src/test/java/org/opensearch/cluster/routing/GroupShardsIteratorTests.java b/server/src/test/java/org/opensearch/cluster/routing/GroupShardsIteratorTests.java index fb9e356b3420d..1cb27ebc77fdd 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/GroupShardsIteratorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/GroupShardsIteratorTests.java @@ -35,8 +35,8 @@ import org.apache.lucene.util.CollectionUtil; import org.opensearch.action.OriginalIndicesTests; import org.opensearch.action.search.SearchShardIterator; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java index 5a8b82c49242a..ebb7529d3f733 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java @@ -32,8 +32,8 @@ package org.opensearch.cluster.routing; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java index 55885fb61ee0c..fd1cb63d61d82 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java @@ -45,9 +45,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.ResponseCollectorService; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/routing/PlainShardIteratorTests.java b/server/src/test/java/org/opensearch/cluster/routing/PlainShardIteratorTests.java index 7548ee8f24e94..7aa61ad8d9819 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/PlainShardIteratorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/PlainShardIteratorTests.java @@ -32,7 +32,7 @@ package org.opensearch.cluster.routing; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingNodeTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingNodeTests.java index 5bd5b7d9f6a67..578d537653684 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingNodeTests.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.net.InetAddress; diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableGenerator.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableGenerator.java index e67fe47d0fb67..7c6752eed418a 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableGenerator.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableGenerator.java @@ -35,7 +35,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import static org.opensearch.cluster.health.ClusterShardHealth.getInactivePrimaryHealth; diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java index 133a334ee2c20..6b869ffed7d23 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java @@ -43,9 +43,9 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.junit.Before; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/cluster/routing/ShardRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/ShardRoutingTests.java index a914ef5da31ca..72425eaad11b9 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/ShardRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/ShardRoutingTests.java @@ -34,8 +34,8 @@ import org.opensearch.Version; import org.opensearch.common.UUIDs; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.SnapshotId; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java b/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java index 6071d2cbc214d..28bf99deec6f4 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java @@ -46,11 +46,11 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.routing.allocation.FailedShard; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationCommandsTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationCommandsTests.java index 281639680f795..98614c517c811 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationCommandsTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationCommandsTests.java @@ -59,17 +59,17 @@ import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.snapshots.SnapshotShardSizeInfo; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java index b1024b56d6cd3..06acfceedd30a 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java @@ -59,7 +59,7 @@ import org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.EmptySnapshotsInfoService; import org.opensearch.test.gateway.TestGatewayAllocator; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 3e21f6c19e150..a8f8296cf9591 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -51,7 +51,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.MockLogAppender; import org.opensearch.test.junit.annotations.TestLogging; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java index 49f7e097c08c2..f2dc745ad33bf 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.VersionUtils; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java index 15dcae65ce6e7..cf32d2b3cf00f 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java @@ -16,7 +16,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.VersionUtils; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java index 546c9dbac208a..557d7db142671 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java @@ -65,8 +65,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.EmptySnapshotsInfoService; import org.opensearch.snapshots.InternalSnapshotsInfoService; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java index 2d4422fdf6ced..afc899469a19c 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java @@ -49,8 +49,8 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.cluster.routing.allocation.decider.ResizeAllocationDecider; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.EmptySnapshotsInfoService; import org.opensearch.test.gateway.TestGatewayAllocator; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/SameShardRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/SameShardRoutingTests.java index cd19f78d54be5..3bee0d994ebbc 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/SameShardRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/SameShardRoutingTests.java @@ -55,7 +55,7 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.snapshots.SnapshotShardSizeInfo; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/StartedShardsRoutingTests.java index d366d63ba8e6b..786f01a783f09 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/StartedShardsRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/StartedShardsRoutingTests.java @@ -47,8 +47,8 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.ArrayList; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java index d1f92e90ab232..e4bbe3c71d918 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -56,8 +56,8 @@ import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.InternalSnapshotsInfoService; import org.opensearch.snapshots.Snapshot; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecidersTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecidersTests.java index 7cfc9440a72e1..7a7a60846443a 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecidersTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecidersTests.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.RoutingAllocation; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 4ccf0a9bc3a20..de155f3fdb896 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -68,8 +68,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.EmptySnapshotsInfoService; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 62c52e93aad33..622d3a2c920b5 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -56,8 +56,8 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import java.util.Collections; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java index d13a63ade6b9e..8cca157210c37 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java @@ -52,7 +52,7 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.common.UUIDs; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; diff --git a/server/src/test/java/org/opensearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/opensearch/cluster/serialization/ClusterSerializationTests.java index ee5615fb94b54..85cedef39b42a 100644 --- a/server/src/test/java/org/opensearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/opensearch/cluster/serialization/ClusterSerializationTests.java @@ -51,10 +51,10 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.snapshots.Snapshot; diff --git a/server/src/test/java/org/opensearch/cluster/serialization/DiffableTests.java b/server/src/test/java/org/opensearch/cluster/serialization/DiffableTests.java index bafa733c05ce4..ac2470e940916 100644 --- a/server/src/test/java/org/opensearch/cluster/serialization/DiffableTests.java +++ b/server/src/test/java/org/opensearch/cluster/serialization/DiffableTests.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.DiffableUtils.MapDiff; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.set.Sets; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterManagerThrottlingStatsTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterManagerThrottlingStatsTests.java index 412073563bbbe..0f8d8ee915c45 100644 --- a/server/src/test/java/org/opensearch/cluster/service/ClusterManagerThrottlingStatsTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/ClusterManagerThrottlingStatsTests.java @@ -8,7 +8,7 @@ package org.opensearch.cluster.service; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; public class ClusterManagerThrottlingStatsTests extends AbstractWireSerializingTestCase { diff --git a/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java b/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java index e24e8b2ad3f87..954be34695009 100644 --- a/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java +++ b/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java @@ -57,7 +57,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexModule; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/common/ChannelsTests.java b/server/src/test/java/org/opensearch/common/ChannelsTests.java index d6bf48b573722..801ac5bae48c5 100644 --- a/server/src/test/java/org/opensearch/common/ChannelsTests.java +++ b/server/src/test/java/org/opensearch/common/ChannelsTests.java @@ -32,8 +32,8 @@ package org.opensearch.common; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.Channels; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java b/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java index 610ad32f0ce35..2d895f0a10412 100644 --- a/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java +++ b/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java @@ -33,7 +33,7 @@ import com.carrotsearch.hppc.ObjectLongHashMap; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/RoundingWireTests.java b/server/src/test/java/org/opensearch/common/RoundingWireTests.java index 12de67d3a56c2..6f6e2a95950e5 100644 --- a/server/src/test/java/org/opensearch/common/RoundingWireTests.java +++ b/server/src/test/java/org/opensearch/common/RoundingWireTests.java @@ -33,7 +33,7 @@ package org.opensearch.common; import org.opensearch.common.Rounding.DateTimeUnit; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/common/bytes/ByteBuffersBytesReferenceTests.java b/server/src/test/java/org/opensearch/common/bytes/ByteBuffersBytesReferenceTests.java index 4665a8e113ff2..02b99c3c4f649 100644 --- a/server/src/test/java/org/opensearch/common/bytes/ByteBuffersBytesReferenceTests.java +++ b/server/src/test/java/org/opensearch/common/bytes/ByteBuffersBytesReferenceTests.java @@ -11,6 +11,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.hamcrest.Matchers; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/server/src/test/java/org/opensearch/common/bytes/BytesArrayTests.java b/server/src/test/java/org/opensearch/common/bytes/BytesArrayTests.java index 9d3fd038f7d38..754e51cb5d130 100644 --- a/server/src/test/java/org/opensearch/common/bytes/BytesArrayTests.java +++ b/server/src/test/java/org/opensearch/common/bytes/BytesArrayTests.java @@ -33,6 +33,8 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.hamcrest.Matchers; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/bytes/CompositeBytesReferenceTests.java b/server/src/test/java/org/opensearch/common/bytes/CompositeBytesReferenceTests.java index 0affae3b53465..2a692d345b4f9 100644 --- a/server/src/test/java/org/opensearch/common/bytes/CompositeBytesReferenceTests.java +++ b/server/src/test/java/org/opensearch/common/bytes/CompositeBytesReferenceTests.java @@ -37,6 +37,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.hamcrest.Matchers; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/common/bytes/PagedBytesReferenceTests.java b/server/src/test/java/org/opensearch/common/bytes/PagedBytesReferenceTests.java index accbcb4b2d192..4ea35b2a0175f 100644 --- a/server/src/test/java/org/opensearch/common/bytes/PagedBytesReferenceTests.java +++ b/server/src/test/java/org/opensearch/common/bytes/PagedBytesReferenceTests.java @@ -33,8 +33,11 @@ package org.opensearch.common.bytes; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.hamcrest.Matchers; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.PagedBytesReference; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/bytes/ReleasableBytesReferenceTests.java b/server/src/test/java/org/opensearch/common/bytes/ReleasableBytesReferenceTests.java index 6b8ffdbd9b6ec..0310ccb283635 100644 --- a/server/src/test/java/org/opensearch/common/bytes/ReleasableBytesReferenceTests.java +++ b/server/src/test/java/org/opensearch/common/bytes/ReleasableBytesReferenceTests.java @@ -34,8 +34,11 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.hamcrest.Matchers; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/common/compress/DeflateCompressedXContentTests.java b/server/src/test/java/org/opensearch/common/compress/DeflateCompressedXContentTests.java index a14fabc30050d..eacbcaced2dcd 100644 --- a/server/src/test/java/org/opensearch/common/compress/DeflateCompressedXContentTests.java +++ b/server/src/test/java/org/opensearch/common/compress/DeflateCompressedXContentTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.compress; import org.apache.lucene.tests.util.TestUtil; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; import org.junit.Assert; diff --git a/server/src/test/java/org/opensearch/common/geo/GeoDistanceTests.java b/server/src/test/java/org/opensearch/common/geo/GeoDistanceTests.java index 033f02f9a8f7d..d78fd27a8172b 100644 --- a/server/src/test/java/org/opensearch/common/geo/GeoDistanceTests.java +++ b/server/src/test/java/org/opensearch/common/geo/GeoDistanceTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.geo; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/geo/GeoJsonSerializationTests.java b/server/src/test/java/org/opensearch/common/geo/GeoJsonSerializationTests.java index 21f10aeb91542..23855c8f8dc91 100644 --- a/server/src/test/java/org/opensearch/common/geo/GeoJsonSerializationTests.java +++ b/server/src/test/java/org/opensearch/common/geo/GeoJsonSerializationTests.java @@ -32,8 +32,8 @@ package org.opensearch.common.geo; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/common/geo/GeoUtilTests.java b/server/src/test/java/org/opensearch/common/geo/GeoUtilTests.java index f94fbb58311bf..3960498445138 100644 --- a/server/src/test/java/org/opensearch/common/geo/GeoUtilTests.java +++ b/server/src/test/java/org/opensearch/common/geo/GeoUtilTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.geo; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/server/src/test/java/org/opensearch/common/geo/GeometryIOTests.java b/server/src/test/java/org/opensearch/common/geo/GeometryIOTests.java index d24a538f5dd16..698697b789a30 100644 --- a/server/src/test/java/org/opensearch/common/geo/GeometryIOTests.java +++ b/server/src/test/java/org/opensearch/common/geo/GeometryIOTests.java @@ -34,9 +34,9 @@ import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.geometry.Geometry; import org.opensearch.geometry.GeometryCollection; import org.opensearch.geometry.ShapeType; diff --git a/server/src/test/java/org/opensearch/common/geo/ShapeRelationTests.java b/server/src/test/java/org/opensearch/common/geo/ShapeRelationTests.java index d006d7d8fff28..42a4fe08b5273 100644 --- a/server/src/test/java/org/opensearch/common/geo/ShapeRelationTests.java +++ b/server/src/test/java/org/opensearch/common/geo/ShapeRelationTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.geo; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/geo/SpatialStrategyTests.java b/server/src/test/java/org/opensearch/common/geo/SpatialStrategyTests.java index 9860a593bc589..bfe1c7cea247f 100644 --- a/server/src/test/java/org/opensearch/common/geo/SpatialStrategyTests.java +++ b/server/src/test/java/org/opensearch/common/geo/SpatialStrategyTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.geo; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java index cc5076d695dd9..417a79da68ae1 100644 --- a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -34,9 +34,9 @@ import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/common/io/VersionedCodecStreamWrapperTests.java b/server/src/test/java/org/opensearch/common/io/VersionedCodecStreamWrapperTests.java index a92b4823466e3..2cca12ca38fa6 100644 --- a/server/src/test/java/org/opensearch/common/io/VersionedCodecStreamWrapperTests.java +++ b/server/src/test/java/org/opensearch/common/io/VersionedCodecStreamWrapperTests.java @@ -17,7 +17,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.OutputStreamIndexOutput; import org.junit.Before; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/io/stream/AbstractWriteableEnumTestCase.java b/server/src/test/java/org/opensearch/common/io/stream/AbstractWriteableEnumTestCase.java index 2786c63e8b4a4..82c9a2dc2bb98 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/AbstractWriteableEnumTestCase.java +++ b/server/src/test/java/org/opensearch/common/io/stream/AbstractWriteableEnumTestCase.java @@ -32,6 +32,9 @@ package org.opensearch.common.io.stream; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/io/stream/BytesStreamsTests.java b/server/src/test/java/org/opensearch/common/io/stream/BytesStreamsTests.java index 111354f2b14db..40b3bb05df974 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/BytesStreamsTests.java +++ b/server/src/test/java/org/opensearch/common/io/stream/BytesStreamsTests.java @@ -35,13 +35,19 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.joda.Joda; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.script.JodaCompatibleZonedDateTime; import org.opensearch.test.OpenSearchTestCase; import org.joda.time.DateTime; diff --git a/server/src/test/java/org/opensearch/common/io/stream/DelayableWriteableTests.java b/server/src/test/java/org/opensearch/common/io/stream/DelayableWriteableTests.java index b0aeb6a86ae2d..ed85d1f33aade 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/DelayableWriteableTests.java +++ b/server/src/test/java/org/opensearch/common/io/stream/DelayableWriteableTests.java @@ -38,6 +38,11 @@ import java.io.IOException; import org.opensearch.Version; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java index 70d3e8d5671e2..ef0bedac72541 100644 --- a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java @@ -81,7 +81,7 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; diff --git a/server/src/test/java/org/opensearch/common/lucene/ShardCoreKeyMapTests.java b/server/src/test/java/org/opensearch/common/lucene/ShardCoreKeyMapTests.java index 8abfb14fe3087..a54047ac69cba 100644 --- a/server/src/test/java/org/opensearch/common/lucene/ShardCoreKeyMapTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/ShardCoreKeyMapTests.java @@ -41,7 +41,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java b/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java index cf4cc5cd56f68..298eeb28fb2e9 100644 --- a/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java @@ -42,7 +42,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; /** Simple tests for this filterreader */ diff --git a/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java index d677c1d21d86c..88627b45efa3c 100644 --- a/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java @@ -46,7 +46,7 @@ import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.VersionFieldMapper; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java index 3a22fed5bb2ec..48a8e5799993d 100644 --- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java +++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.network; import org.opensearch.common.component.AbstractLifecycleComponent; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; diff --git a/server/src/test/java/org/opensearch/common/settings/SecureStringTests.java b/server/src/test/java/org/opensearch/common/settings/SecureStringTests.java index 519def6c244aa..45af07cf1dacd 100644 --- a/server/src/test/java/org/opensearch/common/settings/SecureStringTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SecureStringTests.java @@ -32,6 +32,7 @@ package org.opensearch.common.settings; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index 1c10e19f34a19..3d5a5090cdc82 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -36,9 +36,9 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LogEvent; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.AbstractScopedSettings.SettingUpdater; import org.opensearch.common.settings.Setting.ByteSizeValueParser; diff --git a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java index be81793fd45ae..0c6352d118be2 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java @@ -35,12 +35,13 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java b/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java index c5ea9cb49fd1d..804a53d687c95 100644 --- a/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java @@ -10,8 +10,8 @@ import org.junit.Before; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; diff --git a/server/src/test/java/org/opensearch/common/transport/BoundTransportAddressTests.java b/server/src/test/java/org/opensearch/common/transport/BoundTransportAddressTests.java index 9a7b6c63043cc..a177840059f8f 100644 --- a/server/src/test/java/org/opensearch/common/transport/BoundTransportAddressTests.java +++ b/server/src/test/java/org/opensearch/common/transport/BoundTransportAddressTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.transport; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.net.InetAddress; diff --git a/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java b/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java index 2435fc465a83b..c6bcceec99fbd 100644 --- a/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java +++ b/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.unit; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java b/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java index 7512bf2b32e24..99c1feb78527f 100644 --- a/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java +++ b/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.unit; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.test.AbstractWireSerializingTestCase; import org.hamcrest.MatcherAssert; diff --git a/server/src/test/java/org/opensearch/common/unit/DistanceUnitTests.java b/server/src/test/java/org/opensearch/common/unit/DistanceUnitTests.java index 95fc7f6f7a594..c2dafa82a8699 100644 --- a/server/src/test/java/org/opensearch/common/unit/DistanceUnitTests.java +++ b/server/src/test/java/org/opensearch/common/unit/DistanceUnitTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.unit; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.closeTo; diff --git a/server/src/test/java/org/opensearch/common/unit/FuzzinessTests.java b/server/src/test/java/org/opensearch/common/unit/FuzzinessTests.java index 071b89f577c09..4853cec8d6735 100644 --- a/server/src/test/java/org/opensearch/common/unit/FuzzinessTests.java +++ b/server/src/test/java/org/opensearch/common/unit/FuzzinessTests.java @@ -33,7 +33,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/util/BigArraysTests.java b/server/src/test/java/org/opensearch/common/util/BigArraysTests.java index 909995a2bc19c..c973a76623452 100644 --- a/server/src/test/java/org/opensearch/common/util/BigArraysTests.java +++ b/server/src/test/java/org/opensearch/common/util/BigArraysTests.java @@ -38,6 +38,8 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.util.BigArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/util/CuckooFilterTests.java b/server/src/test/java/org/opensearch/common/util/CuckooFilterTests.java index 5fa1b61bcf211..bba164c607f0f 100644 --- a/server/src/test/java/org/opensearch/common/util/CuckooFilterTests.java +++ b/server/src/test/java/org/opensearch/common/util/CuckooFilterTests.java @@ -33,7 +33,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.hash.MurmurHash3; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/common/util/SetBackedScalingCuckooFilterTests.java b/server/src/test/java/org/opensearch/common/util/SetBackedScalingCuckooFilterTests.java index b1c1aefa9c162..94b758a82f433 100644 --- a/server/src/test/java/org/opensearch/common/util/SetBackedScalingCuckooFilterTests.java +++ b/server/src/test/java/org/opensearch/common/util/SetBackedScalingCuckooFilterTests.java @@ -33,7 +33,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.hash.MurmurHash3; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java index 6cfc63e5193a8..274b6477e3fbd 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java @@ -41,11 +41,11 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.ParseField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.PathUtils; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java index 70e7a776c5c30..866def49af991 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java @@ -35,8 +35,8 @@ import com.fasterxml.jackson.dataformat.cbor.CBORConstants; import com.fasterxml.jackson.dataformat.smile.SmileConstants; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentParserUtilsTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentParserUtilsTests.java index ad3842b3920d5..4f0d4b7cb63dc 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/XContentParserUtilsTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/XContentParserUtilsTests.java @@ -35,11 +35,12 @@ import org.opensearch.common.CheckedBiConsumer; import org.opensearch.common.CheckedConsumer; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -55,9 +56,9 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureFieldName; -import static org.opensearch.common.xcontent.XContentParserUtils.parseTypedKeysObject; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.parseTypedKeysObject; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.instanceOf; diff --git a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java index 4af333355c2b4..6c97297ce43e7 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java @@ -34,8 +34,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/test/java/org/opensearch/common/xcontent/cbor/CborXContentParserTests.java b/server/src/test/java/org/opensearch/common/xcontent/cbor/CborXContentParserTests.java index 4d1a93ed68d1e..5b53862f98772 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/cbor/CborXContentParserTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/cbor/CborXContentParserTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.xcontent.cbor; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/AbstractFilteringTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/support/AbstractFilteringTestCase.java index a82db09e37de1..8a8e40fc6167a 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/support/AbstractFilteringTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/support/AbstractFilteringTestCase.java @@ -33,7 +33,7 @@ package org.opensearch.common.xcontent.support; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/XContentHelperTests.java b/server/src/test/java/org/opensearch/common/xcontent/support/XContentHelperTests.java index ed4a48bec32cf..652e11c2fd99f 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/support/XContentHelperTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/support/XContentHelperTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.xcontent.support; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/XContentMapValuesTests.java b/server/src/test/java/org/opensearch/common/xcontent/support/XContentMapValuesTests.java index ca7e04cd70584..3077f30c5f57c 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/support/XContentMapValuesTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/support/XContentMapValuesTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.xcontent.support; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index ddebc67a08ae3..c318dd89f573c 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -34,7 +34,7 @@ import org.junit.Assert; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 2a2e95e529c7e..ff29bfb8894db 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.service.ClusterApplier; import org.opensearch.cluster.service.ClusterManagerService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java index ad6b3de3d3243..61aac8247647a 100644 --- a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java +++ b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java @@ -33,7 +33,7 @@ package org.opensearch.discovery; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; diff --git a/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java b/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java index 97820e6197eec..63b79c9b53081 100644 --- a/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java +++ b/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java @@ -10,8 +10,8 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.transport.TransportAddress; import org.opensearch.extensions.DiscoveryExtensionNode; diff --git a/server/src/test/java/org/opensearch/discovery/InitializeExtensionResponseTests.java b/server/src/test/java/org/opensearch/discovery/InitializeExtensionResponseTests.java index ea7cc96b0e2a4..dce868bd395ad 100644 --- a/server/src/test/java/org/opensearch/discovery/InitializeExtensionResponseTests.java +++ b/server/src/test/java/org/opensearch/discovery/InitializeExtensionResponseTests.java @@ -8,8 +8,8 @@ package org.opensearch.discovery; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java index 7e7bb2f0a2911..91eec3d2edfaf 100644 --- a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java +++ b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.node.DiscoveryNodes.Builder; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.discovery.PeerFinder.TransportAddressConnector; diff --git a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java index 60800ee6b64eb..95f84a70986e1 100644 --- a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java +++ b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java index 8a6c49ac36ae5..46f98b7a2755e 100644 --- a/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java @@ -42,9 +42,9 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.MetadataStateFormat; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.Node; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java index b1e701a0dc248..382698a300591 100644 --- a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java +++ b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java @@ -50,7 +50,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.gateway.PersistedClusterStateService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matcher; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionResponseTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionResponseTests.java index de8c677e5bcc6..5b438f6130c76 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionResponseTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionResponseTests.java @@ -8,8 +8,8 @@ package org.opensearch.extensions; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 3f61d01166fb9..415ed4567163c 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -49,10 +49,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -62,6 +59,9 @@ import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.env.Environment; import org.opensearch.extensions.proto.ExtensionRequestProto; import org.opensearch.extensions.rest.RegisterRestActionsRequest; diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionActionRequestTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionActionRequestTests.java index cf7c391abe4e5..57ff3b74f586f 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionActionRequestTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionActionRequestTests.java @@ -9,8 +9,8 @@ package org.opensearch.extensions.action; import com.google.protobuf.ByteString; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionActionResponseTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionActionResponseTests.java index 5ec8c16027da2..07eb721e1bb7f 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionActionResponseTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionActionResponseTests.java @@ -8,8 +8,8 @@ package org.opensearch.extensions.action; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionActionUtilTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionActionUtilTests.java index d2b889d33da9a..d9274a4c45e5b 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionActionUtilTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionActionUtilTests.java @@ -12,11 +12,11 @@ import org.mockito.Mockito; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionHandleTransportRequestTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionHandleTransportRequestTests.java index 70b17273ef690..b8a54392f7388 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionHandleTransportRequestTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionHandleTransportRequestTests.java @@ -9,8 +9,8 @@ package org.opensearch.extensions.action; import com.google.protobuf.ByteString; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java index 759e2e3be9b58..e46e05cd2dbec 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java @@ -17,7 +17,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/test/java/org/opensearch/extensions/action/RegisterTransportActionsRequestTests.java b/server/src/test/java/org/opensearch/extensions/action/RegisterTransportActionsRequestTests.java index fe3c0dc8dd73d..b03f87a5057e0 100644 --- a/server/src/test/java/org/opensearch/extensions/action/RegisterTransportActionsRequestTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/RegisterTransportActionsRequestTests.java @@ -10,7 +10,7 @@ import org.junit.Before; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/extensions/action/RemoteExtensionActionResponseTests.java b/server/src/test/java/org/opensearch/extensions/action/RemoteExtensionActionResponseTests.java index 4ce42450bd577..7ade7a695ebdd 100644 --- a/server/src/test/java/org/opensearch/extensions/action/RemoteExtensionActionResponseTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/RemoteExtensionActionResponseTests.java @@ -8,8 +8,8 @@ package org.opensearch.extensions.action; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/action/TransportActionRequestFromExtensionTests.java b/server/src/test/java/org/opensearch/extensions/action/TransportActionRequestFromExtensionTests.java index c002dfe135e36..ade96ac51ab2b 100644 --- a/server/src/test/java/org/opensearch/extensions/action/TransportActionRequestFromExtensionTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/TransportActionRequestFromExtensionTests.java @@ -9,8 +9,8 @@ package org.opensearch.extensions.action; import com.google.protobuf.ByteString; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java index 0cf1f524a3ac4..e12549b93ab53 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java @@ -8,17 +8,17 @@ package org.opensearch.extensions.rest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest.Method; import org.opensearch.http.HttpRequest; diff --git a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java index cdc79e9778e2d..e76e6b98811f7 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java @@ -14,21 +14,20 @@ import java.util.List; import java.util.Map; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.http.HttpRequest; import org.opensearch.http.HttpResponse; -import org.opensearch.http.HttpRequest.HttpVersion; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestRequest.Method; import org.opensearch.test.OpenSearchTestCase; import static org.opensearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE; -import static org.opensearch.rest.RestStatus.ACCEPTED; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.ACCEPTED; +import static org.opensearch.core.rest.RestStatus.OK; public class ExtensionRestResponseTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/extensions/rest/RegisterRestActionsTests.java b/server/src/test/java/org/opensearch/extensions/rest/RegisterRestActionsTests.java index 414d0bc3eadf1..8cb29bf264db9 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RegisterRestActionsTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RegisterRestActionsTests.java @@ -10,8 +10,8 @@ import java.util.List; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 7dd616c678e74..030df865c4e4a 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -27,8 +27,6 @@ import org.mockito.Mockito; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -38,7 +36,9 @@ import org.opensearch.extensions.ExtensionsSettings; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index 23a9169b91e21..8695e409de0bc 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsModule; diff --git a/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java b/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java index 68f32672871ad..bf9234a7f129d 100644 --- a/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java +++ b/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java @@ -11,8 +11,8 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java index 982c21a9e57ec..31a27503069d7 100644 --- a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java +++ b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java index 1f63dff04b3df..c83da46b23fb1 100644 --- a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java @@ -47,7 +47,7 @@ import org.opensearch.common.settings.SettingUpgrader; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/gateway/DanglingIndicesStateTests.java b/server/src/test/java/org/opensearch/gateway/DanglingIndicesStateTests.java index 1934307f88792..d10ba879fc18e 100644 --- a/server/src/test/java/org/opensearch/gateway/DanglingIndicesStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/DanglingIndicesStateTests.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/gateway/IncrementalClusterStateWriterTests.java b/server/src/test/java/org/opensearch/gateway/IncrementalClusterStateWriterTests.java index 58c6ddbb05328..f3c1108c163b5 100644 --- a/server/src/test/java/org/opensearch/gateway/IncrementalClusterStateWriterTests.java +++ b/server/src/test/java/org/opensearch/gateway/IncrementalClusterStateWriterTests.java @@ -57,7 +57,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.MockLogAppender; import org.opensearch.test.junit.annotations.TestLogging; import org.mockito.ArgumentCaptor; diff --git a/server/src/test/java/org/opensearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/MetaStateServiceTests.java index 7204f6001baa3..4e8f48cc4c1cc 100644 --- a/server/src/test/java/org/opensearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/MetaStateServiceTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java index 2b8cec86db5a2..211c13e00e6db 100644 --- a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java @@ -61,7 +61,7 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeMetadata; import org.opensearch.gateway.PersistedClusterStateService.Writer; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.MockLogAppender; diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java index 9a264781b0644..61bf5f347c2d5 100644 --- a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java @@ -64,7 +64,7 @@ import org.opensearch.env.ShardLockObtainFailedException; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; diff --git a/server/src/test/java/org/opensearch/gateway/PriorityComparatorTests.java b/server/src/test/java/org/opensearch/gateway/PriorityComparatorTests.java index 9e5c698091dde..048579d767dae 100644 --- a/server/src/test/java/org/opensearch/gateway/PriorityComparatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/PriorityComparatorTests.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/gateway/ReplicaShardAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/ReplicaShardAllocatorTests.java index 36ac93524d6aa..968a0474051f7 100644 --- a/server/src/test/java/org/opensearch/gateway/ReplicaShardAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/ReplicaShardAllocatorTests.java @@ -62,7 +62,7 @@ import org.opensearch.index.engine.Engine; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata; diff --git a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java index ed82aa361d45d..b36b3110b2123 100644 --- a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java +++ b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.network.NetworkService; import org.opensearch.common.network.NetworkUtils; import org.opensearch.common.settings.ClusterSettings; @@ -50,7 +50,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.MockLogAppender; diff --git a/server/src/test/java/org/opensearch/http/CorsHandlerTests.java b/server/src/test/java/org/opensearch/http/CorsHandlerTests.java index 7b38d7039dc0e..a6be3ef0828fc 100644 --- a/server/src/test/java/org/opensearch/http/CorsHandlerTests.java +++ b/server/src/test/java/org/opensearch/http/CorsHandlerTests.java @@ -32,12 +32,12 @@ package org.opensearch.http; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.core.common.Strings; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java index c0f6a9ba68383..0338c9009ab86 100644 --- a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java +++ b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java @@ -33,14 +33,14 @@ package org.opensearch.http; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.xcontent.json.JsonXContent; @@ -51,7 +51,7 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/http/TestHttpRequest.java b/server/src/test/java/org/opensearch/http/TestHttpRequest.java index 3b604ba03aa6f..57b107c5300ea 100644 --- a/server/src/test/java/org/opensearch/http/TestHttpRequest.java +++ b/server/src/test/java/org/opensearch/http/TestHttpRequest.java @@ -32,10 +32,10 @@ package org.opensearch.http; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.Arrays; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/http/TestHttpResponse.java b/server/src/test/java/org/opensearch/http/TestHttpResponse.java index 6dc82cb310cc6..9afe8591badc6 100644 --- a/server/src/test/java/org/opensearch/http/TestHttpResponse.java +++ b/server/src/test/java/org/opensearch/http/TestHttpResponse.java @@ -32,8 +32,8 @@ package org.opensearch.http; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import java.util.ArrayList; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 32b8fb5a4dc62..95d0ae706b286 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -66,6 +66,7 @@ import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.Index; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; @@ -86,7 +87,7 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.SearchOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.NonNegativeScoresSimilarity; import org.opensearch.index.similarity.SimilarityService; diff --git a/server/src/test/java/org/opensearch/index/IndexServiceTests.java b/server/src/test/java/org/opensearch/index/IndexServiceTests.java index 209b7eb4c5696..99207040207fc 100644 --- a/server/src/test/java/org/opensearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/opensearch/index/IndexServiceTests.java @@ -40,6 +40,7 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/IndexTests.java b/server/src/test/java/org/opensearch/index/IndexTests.java index 0efe25ca3cbcb..5a6bcab775ad4 100644 --- a/server/src/test/java/org/opensearch/index/IndexTests.java +++ b/server/src/test/java/org/opensearch/index/IndexTests.java @@ -34,7 +34,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java b/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java index 055ab63cd03f3..2220434f9b9ed 100644 --- a/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java @@ -21,7 +21,8 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.stats.IndexingPressurePerShardStats; import org.opensearch.index.stats.IndexingPressureStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/opensearch/index/IndexingSlowLogTests.java index 75a346e444b73..6823bb5a8225f 100644 --- a/server/src/test/java/org/opensearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/opensearch/index/IndexingSlowLogTests.java @@ -41,21 +41,22 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.logging.Loggers; import org.opensearch.common.logging.MockAppender; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexingSlowLog.IndexingSlowLogMessage; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.InternalEngineTests; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.Uid; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; diff --git a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java index 4735c3a8ee3cb..b1dba0c22c075 100644 --- a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java @@ -35,7 +35,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/SearchSlowLogTests.java b/server/src/test/java/org/opensearch/index/SearchSlowLogTests.java index ea146ec20b16a..fe902a09a8c7c 100644 --- a/server/src/test/java/org/opensearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/opensearch/index/SearchSlowLogTests.java @@ -45,7 +45,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.internal.SearchContext; import org.opensearch.search.internal.ShardSearchRequest; diff --git a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java index 1ebdd111bfed3..827c304ba9bd9 100644 --- a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java @@ -20,7 +20,7 @@ import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java index e0189c1713124..eedb45e8e96b6 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java @@ -17,7 +17,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.stats.IndexingPressurePerShardStats; import org.opensearch.index.stats.IndexingPressureStats; import org.opensearch.index.stats.ShardIndexingPressureStats; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java index 97ffc7150db9c..023063c7d6e03 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java @@ -11,7 +11,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.util.concurrent.TimeUnit; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java index 758345e719e6d..b871c8fb8f364 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java @@ -12,7 +12,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.util.Map; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java index 77593660a7d12..e7600b1d4c41a 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java @@ -13,7 +13,8 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.stats.IndexingPressurePerShardStats; import org.opensearch.index.stats.IndexingPressureStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTrackerTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTrackerTests.java index d73a9e2f4a810..686844794809c 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTrackerTests.java @@ -13,7 +13,7 @@ import org.opensearch.index.ShardIndexingPressureTracker.StatsTracker; import org.opensearch.index.ShardIndexingPressureTracker.RejectionTracker; import org.opensearch.index.ShardIndexingPressureTracker.PerformanceTracker; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; public class ShardIndexingPressureTrackerTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/index/analysis/ReloadableCustomAnalyzerTests.java b/server/src/test/java/org/opensearch/index/analysis/ReloadableCustomAnalyzerTests.java index eb057fae66ddf..4cbaef5ab82df 100644 --- a/server/src/test/java/org/opensearch/index/analysis/ReloadableCustomAnalyzerTests.java +++ b/server/src/test/java/org/opensearch/index/analysis/ReloadableCustomAnalyzerTests.java @@ -39,7 +39,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; import org.junit.BeforeClass; diff --git a/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java b/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java index 38b5bf8d81f66..501a8b01c3899 100644 --- a/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java +++ b/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 209c9925bb06d..612bebf74bc70 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -99,8 +99,8 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.TriFunction; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.logging.Loggers; @@ -139,7 +139,7 @@ import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardUtils; import org.opensearch.index.store.Store; import org.opensearch.index.translog.DefaultTranslogDeletionPolicy; diff --git a/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java b/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java index b23394d1edd9c..808bf2056dabe 100644 --- a/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.tests.util.TestUtil; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; diff --git a/server/src/test/java/org/opensearch/index/engine/SegmentTests.java b/server/src/test/java/org/opensearch/index/engine/SegmentTests.java index 7b0072f83f9cb..77dfccd353ede 100644 --- a/server/src/test/java/org/opensearch/index/engine/SegmentTests.java +++ b/server/src/test/java/org/opensearch/index/engine/SegmentTests.java @@ -40,7 +40,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.util.Version; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataTestCase.java index 10ccc99f44603..a2eea22e920d9 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataTestCase.java @@ -60,7 +60,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.plugins.Plugin; diff --git a/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java index 76496491b3ed4..9bf88913a2895 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -66,7 +66,7 @@ import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.opensearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.MultiValueMode; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/fielddata/BinaryDVFieldDataTests.java b/server/src/test/java/org/opensearch/index/fielddata/BinaryDVFieldDataTests.java index fef7b67da5c34..66c785c3d4334 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/BinaryDVFieldDataTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java b/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java index a5e07c4fba373..48ab9d60727b7 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java @@ -48,7 +48,7 @@ import org.opensearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.opensearch.index.mapper.TextFieldMapper; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/fielddata/FieldDataStatsTests.java b/server/src/test/java/org/opensearch/index/fielddata/FieldDataStatsTests.java index 157b1719a883c..296f760095ed1 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/FieldDataStatsTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/FieldDataStatsTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.FieldMemoryStatsTests; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java index 3fe03b04663cf..ac8691855db1a 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/IndexFieldDataServiceTests.java @@ -56,7 +56,7 @@ import org.opensearch.index.mapper.Mapper.BuilderContext; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.TextFieldMapper; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.plugins.Plugin; diff --git a/server/src/test/java/org/opensearch/index/fieldstats/FieldStatsProviderRefreshTests.java b/server/src/test/java/org/opensearch/index/fieldstats/FieldStatsProviderRefreshTests.java index 35c16c9954622..286c2a805692d 100644 --- a/server/src/test/java/org/opensearch/index/fieldstats/FieldStatsProviderRefreshTests.java +++ b/server/src/test/java/org/opensearch/index/fieldstats/FieldStatsProviderRefreshTests.java @@ -40,7 +40,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesRequestCache; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchSingleNodeTestCase; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; diff --git a/server/src/test/java/org/opensearch/index/get/DocumentFieldTests.java b/server/src/test/java/org/opensearch/index/get/DocumentFieldTests.java index 06893c216e776..d3f6fcbe5b54c 100644 --- a/server/src/test/java/org/opensearch/index/get/DocumentFieldTests.java +++ b/server/src/test/java/org/opensearch/index/get/DocumentFieldTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.get; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/index/get/GetResultTests.java b/server/src/test/java/org/opensearch/index/get/GetResultTests.java index 61de3f9030c33..3796a71bf58bb 100644 --- a/server/src/test/java/org/opensearch/index/get/GetResultTests.java +++ b/server/src/test/java/org/opensearch/index/get/GetResultTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.get; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -62,7 +62,7 @@ import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.index.get.DocumentFieldTests.randomDocumentField; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; diff --git a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java index 878fd48e3fb02..40e7786e829c6 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java @@ -33,8 +33,8 @@ package org.opensearch.index.mapper; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java index 1f99c855ce806..a5e3e7bd363d7 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java @@ -47,7 +47,7 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java index 5fe1a979ef1e7..ce91d45029517 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/index/mapper/DataStreamFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DataStreamFieldMapperTests.java index 374b7ac9a5271..50edd29b4b4c9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DataStreamFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DataStreamFieldMapperTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index 6d69e62bc7dd8..a9ac83790226f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -37,8 +37,8 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java index 222cdf9f7684c..365d3738f4022 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java @@ -33,7 +33,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FieldNamesFieldMapperTests.java index 639de9d314641..d3001c5c9c890 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FieldNamesFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FieldNamesFieldMapperTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java index 54393b10a3c5d..a2edff295d8f2 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java @@ -9,7 +9,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/GenericStoreDynamicTemplateTests.java b/server/src/test/java/org/opensearch/index/mapper/GenericStoreDynamicTemplateTests.java index 9c9c0440231de..1834b087e3bdd 100644 --- a/server/src/test/java/org/opensearch/index/mapper/GenericStoreDynamicTemplateTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/GenericStoreDynamicTemplateTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.ParseContext.Document; diff --git a/server/src/test/java/org/opensearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IdFieldMapperTests.java index 93d0596a9bfb5..ec30b014c9359 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IdFieldMapperTests.java @@ -35,8 +35,8 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IndexFieldMapperTests.java index c4225cb576550..1d1ff0405e2ee 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IndexFieldMapperTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IpRangeFieldMapperTests.java index e5c8e822b2f56..a805f1b235856 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpRangeFieldMapperTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.network.InetAddresses; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/mapper/JavaMultiFieldMergeTests.java b/server/src/test/java/org/opensearch/index/mapper/JavaMultiFieldMergeTests.java index 7e00a463124f1..f0bc7c9f8c616 100644 --- a/server/src/test/java/org/opensearch/index/mapper/JavaMultiFieldMergeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/JavaMultiFieldMergeTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java index 01a901b3cd5cb..c1c1b6d9925b5 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.analysis.TokenStream; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/mapper/MultiFieldTests.java b/server/src/test/java/org/opensearch/index/mapper/MultiFieldTests.java index 5b1c060a827ac..76110a8a23e1f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MultiFieldTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MultiFieldTests.java @@ -36,8 +36,8 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java index 7722e7c07d8a9..d5b22ae64cd54 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java index 6ad1d0f7f09b9..9823c54b4ab13 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/NullValueObjectMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/NullValueObjectMappingTests.java index 95c21823bfcae..16c4a172e46ba 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NullValueObjectMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NullValueObjectMappingTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java index 247ea64b74c3d..ee2380ff2c04d 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.mapper.NumberFieldMapper.NumberType; diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index 07b0ad822209c..403b157ecd22c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/PathMatchDynamicTemplateTests.java b/server/src/test/java/org/opensearch/index/mapper/PathMatchDynamicTemplateTests.java index e98dc399b3b41..36f63f9d90ab0 100644 --- a/server/src/test/java/org/opensearch/index/mapper/PathMatchDynamicTemplateTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/PathMatchDynamicTemplateTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.ParseContext.Document; diff --git a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java index 92236ad34013b..58029df33a2ce 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java index 6925faa44b556..518e5e880f4f8 100644 --- a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java @@ -34,8 +34,8 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java b/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java index 3a608ebe828e1..0984cfb2d1755 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.ByteBuffersDirectory; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.util.set.Sets; diff --git a/server/src/test/java/org/opensearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/opensearch/index/mapper/TypeParsersTests.java index 5b536a5756c95..98493fe87ff07 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TypeParsersTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java index cd22f72c8ffbc..8651f973b3317 100644 --- a/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java @@ -35,7 +35,7 @@ import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/query/AbstractQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/AbstractQueryBuilderTests.java index 510323be8fdbb..8902143b708e3 100644 --- a/server/src/test/java/org/opensearch/index/query/AbstractQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/AbstractQueryBuilderTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/index/query/CombineFunctionTests.java b/server/src/test/java/org/opensearch/index/query/CombineFunctionTests.java index 31981cce0de91..d7243f4c024f7 100644 --- a/server/src/test/java/org/opensearch/index/query/CombineFunctionTests.java +++ b/server/src/test/java/org/opensearch/index/query/CombineFunctionTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.query; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/CombineIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/CombineIntervalsSourceProviderTests.java index 58e49fc1269f0..f1a7cc4e5cfeb 100644 --- a/server/src/test/java/org/opensearch/index/query/CombineIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/CombineIntervalsSourceProviderTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchModule; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java index b09dc097bd4fd..3b5fd705fcd08 100644 --- a/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java index b3769e599e65a..2bfe964ce7259 100644 --- a/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/DisjunctionIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/DisjunctionIntervalsSourceProviderTests.java index 4d2898d3f01f1..3626b1f4c46d9 100644 --- a/server/src/test/java/org/opensearch/index/query/DisjunctionIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/DisjunctionIntervalsSourceProviderTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchModule; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java index 402b44ed3df76..fb5fc4192d02c 100644 --- a/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/FilterIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/FilterIntervalsSourceProviderTests.java index 9c063eb1171ad..2835647ada0b6 100644 --- a/server/src/test/java/org/opensearch/index/query/FilterIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/FilterIntervalsSourceProviderTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/server/src/test/java/org/opensearch/index/query/FuzzyIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/FuzzyIntervalsSourceProviderTests.java index 6d4fb26e8cb4c..d25d34ed80be2 100644 --- a/server/src/test/java/org/opensearch/index/query/FuzzyIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/FuzzyIntervalsSourceProviderTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.IntervalsSourceProvider.Fuzzy; diff --git a/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java index 5e98a019d0cd8..b98f3f561d673 100644 --- a/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.unit.Fuzziness; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java index 8ef34fcb8225b..5140f5b5d466f 100644 --- a/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoDistance; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.unit.DistanceUnit; diff --git a/server/src/test/java/org/opensearch/index/query/GeoPolygonQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/GeoPolygonQueryBuilderTests.java index ed569e4d0e315..ea65395fbea13 100644 --- a/server/src/test/java/org/opensearch/index/query/GeoPolygonQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/GeoPolygonQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/test/java/org/opensearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/GeoShapeQueryBuilderTests.java index 22bef292caa3e..e73f2e3f26928 100644 --- a/server/src/test/java/org/opensearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/GeoShapeQueryBuilderTests.java @@ -41,7 +41,7 @@ import org.opensearch.action.get.GetResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.geo.builders.EnvelopeBuilder; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/test/java/org/opensearch/index/query/IdsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/IdsQueryBuilderTests.java index 8f6584fee5dd7..40f4d762f18be 100644 --- a/server/src/test/java/org/opensearch/index/query/IdsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/IdsQueryBuilderTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java index d252d76fa19b2..cb75889a0aa24 100644 --- a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/IntervalQueryBuilderTests.java index 8e1899ee0017d..23d6df398a8b2 100644 --- a/server/src/test/java/org/opensearch/index/query/IntervalQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/IntervalQueryBuilderTests.java @@ -44,7 +44,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java index a23b4fd75e8a0..1a311448b7fb5 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java @@ -42,7 +42,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.lucene.search.Queries; import org.opensearch.index.search.MatchQuery; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/MatchIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/MatchIntervalsSourceProviderTests.java index 3f6f390a8e77a..03e5b3d50357e 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchIntervalsSourceProviderTests.java @@ -32,8 +32,8 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchModule; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilderTests.java index 6bff0b74af6d1..93826b52ca179 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilderTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; import org.opensearch.index.search.MatchQuery.ZeroTermsQuery; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java index defe8e335a90f..f8d5d2ce3d062 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java @@ -40,7 +40,7 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.index.search.MatchQuery.ZeroTermsQuery; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java index 9fa81a180e70a..6855a14ac5a8d 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java @@ -55,7 +55,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; diff --git a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java index 978aa0aa3b933..e6ec132a10474 100644 --- a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -46,7 +46,7 @@ import org.opensearch.action.termvectors.TermVectorsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.search.MoreLikeThisQuery; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java index 735bb4d47fe47..c71bbba93a4fb 100644 --- a/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java @@ -48,7 +48,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; diff --git a/server/src/test/java/org/opensearch/index/query/OperatorTests.java b/server/src/test/java/org/opensearch/index/query/OperatorTests.java index a6293cd833099..5d90338fe5278 100644 --- a/server/src/test/java/org/opensearch/index/query/OperatorTests.java +++ b/server/src/test/java/org/opensearch/index/query/OperatorTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.BooleanClause; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/index/query/PrefixIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/PrefixIntervalsSourceProviderTests.java index fa290d42c8357..93dd229783805 100644 --- a/server/src/test/java/org/opensearch/index/query/PrefixIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/PrefixIntervalsSourceProviderTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java index 8f4f70e96e2b4..52093e65d0122 100644 --- a/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java index aca32a10e49aa..2e499103540fe 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java @@ -49,8 +49,8 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/index/query/QueryShardExceptionTests.java b/server/src/test/java/org/opensearch/index/query/QueryShardExceptionTests.java index da4bfa9f1c390..d7549d8c77340 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryShardExceptionTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryShardExceptionTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index 22c10844028a9..0e38f6ad94d26 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -45,7 +45,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.index.mapper.DateFieldMapper; diff --git a/server/src/test/java/org/opensearch/index/query/RegexpIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/RegexpIntervalsSourceProviderTests.java index 8217aedac9013..0eb5ea7589133 100644 --- a/server/src/test/java/org/opensearch/index/query/RegexpIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RegexpIntervalsSourceProviderTests.java @@ -11,7 +11,7 @@ import static org.opensearch.index.query.IntervalsSourceProvider.Regexp; import static org.opensearch.index.query.IntervalsSourceProvider.fromXContent; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java index 6366951329788..30c8c07fa2c27 100644 --- a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/ScoreModeTests.java b/server/src/test/java/org/opensearch/index/query/ScoreModeTests.java index f6be1868b49e5..6b699f2d40d1f 100644 --- a/server/src/test/java/org/opensearch/index/query/ScoreModeTests.java +++ b/server/src/test/java/org/opensearch/index/query/ScoreModeTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.query; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/ScriptQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/ScriptQueryBuilderTests.java index 9b37bdd3e7d1c..446cb25d8bb97 100644 --- a/server/src/test/java/org/opensearch/index/query/ScriptQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/ScriptQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.search.Query; import org.opensearch.OpenSearchException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; diff --git a/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java index 33be236b45ab6..fff4369e155b3 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.queries.spans.SpanContainingQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java index 1d0348e35c68b..2733a500fd464 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.queries.spans.SpanFirstQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java index 145c3ca7cbb1d..bc514208c803d 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -54,7 +54,7 @@ import org.apache.lucene.store.Directory; import org.opensearch.common.Strings; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java index 416e37db764b3..8c9130d4b7bbd 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java index 1b75763c82e4d..8847aa9882a20 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.queries.spans.SpanNotQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java index 45764708efb46..45323b5df74df 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanTermQueryBuilderTests.java index 30f2143d74f1d..bacaaa150ec93 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanTermQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanTermQueryBuilderTests.java @@ -37,7 +37,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java index 25fd137aac286..e8b6a21254ff8 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.queries.spans.SpanWithinQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermQueryBuilderTests.java index cc877c7590c6a..b42ac35ac9407 100644 --- a/server/src/test/java/org/opensearch/index/query/TermQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/TermQueryBuilderTests.java @@ -39,7 +39,7 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.index.mapper.MappedFieldType; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java index 7b426864556be..3ee04450e4e73 100644 --- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java @@ -42,9 +42,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/query/WildcardIntervalsSourceProviderTests.java b/server/src/test/java/org/opensearch/index/query/WildcardIntervalsSourceProviderTests.java index 5b0e5ef33fa69..d397cf7db8500 100644 --- a/server/src/test/java/org/opensearch/index/query/WildcardIntervalsSourceProviderTests.java +++ b/server/src/test/java/org/opensearch/index/query/WildcardIntervalsSourceProviderTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.query; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java index 9e99b7667f3e0..f4ec241a64a26 100644 --- a/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.WildcardQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/WrapperQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/WrapperQueryBuilderTests.java index 09944b94aca8c..09605bcd0a8dd 100644 --- a/server/src/test/java/org/opensearch/index/query/WrapperQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/WrapperQueryBuilderTests.java @@ -37,9 +37,9 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java b/server/src/test/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java index eac05176e9766..b283062de11b7 100644 --- a/server/src/test/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java +++ b/server/src/test/java/org/opensearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.query.functionscore; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java index 9eededb7eef4c..8e4ae9c6cfede 100644 --- a/server/src/test/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java @@ -39,11 +39,11 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; diff --git a/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java b/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java index 5da29dadf07a5..2b0aaa2dd7091 100644 --- a/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java +++ b/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.index.query.plugin; import org.apache.lucene.search.Query; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; diff --git a/server/src/test/java/org/opensearch/index/refresh/RefreshStatsTests.java b/server/src/test/java/org/opensearch/index/refresh/RefreshStatsTests.java index 6b04e0ab43d8c..3a1239a21c889 100644 --- a/server/src/test/java/org/opensearch/index/refresh/RefreshStatsTests.java +++ b/server/src/test/java/org/opensearch/index/refresh/RefreshStatsTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.refresh; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/reindex/BulkByScrollResponseTests.java b/server/src/test/java/org/opensearch/index/reindex/BulkByScrollResponseTests.java index f884000fafbd2..1677416249bd2 100644 --- a/server/src/test/java/org/opensearch/index/reindex/BulkByScrollResponseTests.java +++ b/server/src/test/java/org/opensearch/index/reindex/BulkByScrollResponseTests.java @@ -38,7 +38,7 @@ import org.opensearch.action.bulk.BulkItemResponse.Failure; import org.opensearch.client.transport.NoNodeAvailableException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.reindex.BulkByScrollTask.Status; diff --git a/server/src/test/java/org/opensearch/index/reindex/ReindexRequestTests.java b/server/src/test/java/org/opensearch/index/reindex/ReindexRequestTests.java index 44fb838cfc647..f65aaa0a30d96 100644 --- a/server/src/test/java/org/opensearch/index/reindex/ReindexRequestTests.java +++ b/server/src/test/java/org/opensearch/index/reindex/ReindexRequestTests.java @@ -33,9 +33,9 @@ package org.opensearch.index.reindex; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java index 1bab4bbfd9d31..5ccacd4048596 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java @@ -15,7 +15,7 @@ import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java index 2ee254b10facc..badfeb0d67c05 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java @@ -10,10 +10,10 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java index b733f81fdb701..b61150a9a81e2 100644 --- a/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java b/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java index 143d11bbbf13f..b4d21a360bc9c 100644 --- a/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java +++ b/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java @@ -44,7 +44,7 @@ import org.opensearch.index.seqno.RetentionLeaseUtils; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.VersionUtils; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/index/search/geo/GeoPointParsingTests.java b/server/src/test/java/org/opensearch/index/search/geo/GeoPointParsingTests.java index 16d5fb9c404b3..93075cefa5570 100644 --- a/server/src/test/java/org/opensearch/index/search/geo/GeoPointParsingTests.java +++ b/server/src/test/java/org/opensearch/index/search/geo/GeoPointParsingTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.search.geo; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/search/nested/AbstractNumberNestedSortingTestCase.java b/server/src/test/java/org/opensearch/index/search/nested/AbstractNumberNestedSortingTestCase.java index 26fd18c9fb558..85cefbb920a5b 100644 --- a/server/src/test/java/org/opensearch/index/search/nested/AbstractNumberNestedSortingTestCase.java +++ b/server/src/test/java/org/opensearch/index/search/nested/AbstractNumberNestedSortingTestCase.java @@ -55,7 +55,7 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.MultiValueMode; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java index 7f130680f83e5..f3ae66bdbf1ab 100644 --- a/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java @@ -75,7 +75,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.MultiValueMode; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.NestedSortBuilder; diff --git a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java index 18af6ff55c412..4832625b315c1 100644 --- a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java @@ -39,10 +39,10 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java b/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java index b97b537d4280c..4d9c73cd8cb0e 100644 --- a/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.SafeCommitInfo; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.IndexSettingsModule; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index 28d949d7ff2f1..6c9bf879e725a 100644 --- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -42,7 +42,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.WriteStateException; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.IndexSettingsModule; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java index 17a6bfc8fbd82..bac70cddec85c 100644 --- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java +++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java @@ -39,7 +39,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.settings.Settings; import org.opensearch.index.engine.SafeCommitInfo; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java index d26d652451f35..822cae1b9392d 100644 --- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java @@ -43,12 +43,12 @@ import org.opensearch.common.Randomness; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; import org.opensearch.index.IndexSettings; import org.opensearch.index.SegmentReplicationShardStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseActionsTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseActionsTests.java index 936e8a65156c0..2a19a98c1d63a 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseActionsTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseActionsTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.Node; import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java index b65662a53ea91..baffd41d68dea 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java @@ -44,10 +44,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.WriteStateException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsTests.java index 10b1673710fba..45bbc92c65aa5 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java index 83e2c5f3e3f42..c5e79abf53419 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.seqno; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java index 579b8ce1c5aa6..6a87ea1088492 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java @@ -42,11 +42,11 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseTests.java index e5e5a185050e1..2c7714f425956 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.seqno; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/shard/DocsStatsTests.java b/server/src/test/java/org/opensearch/index/shard/DocsStatsTests.java index 8d6d4a6502771..189996d877c4f 100644 --- a/server/src/test/java/org/opensearch/index/shard/DocsStatsTests.java +++ b/server/src/test/java/org/opensearch/index/shard/DocsStatsTests.java @@ -32,9 +32,9 @@ package org.opensearch.index.shard; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java b/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java index aa86b4067d566..68ec1bd9f2814 100644 --- a/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java @@ -38,7 +38,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.Scheduler; import org.junit.After; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java b/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java index f030c3cb21c62..b9dc75adec974 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java @@ -49,6 +49,7 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.engine.Engine; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java index 5e6062a252938..0296465352836 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java @@ -39,6 +39,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 60644ebaf94a3..5ddbf1971f114 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -77,11 +77,11 @@ import org.opensearch.common.Randomness; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.collect.Tuple; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -92,6 +92,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.env.NodeEnvironment; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexingOperationListenerTests.java b/server/src/test/java/org/opensearch/index/shard/IndexingOperationListenerTests.java index 6ac5f96e11f34..3ba22b255a109 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexingOperationListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexingOperationListenerTests.java @@ -32,7 +32,8 @@ package org.opensearch.index.shard; import org.apache.lucene.index.Term; -import org.opensearch.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.InternalEngineTests; import org.opensearch.index.mapper.ParsedDocument; diff --git a/server/src/test/java/org/opensearch/index/shard/NewPathForShardTests.java b/server/src/test/java/org/opensearch/index/shard/NewPathForShardTests.java index 0d2a27aa0714e..36b2504159a19 100644 --- a/server/src/test/java/org/opensearch/index/shard/NewPathForShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/NewPathForShardTests.java @@ -36,6 +36,7 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeEnvironment.NodePath; diff --git a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java index 5e7fbeb6887f8..282f4c09b982d 100644 --- a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java @@ -38,12 +38,12 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java index 3de308022308e..f812312004b21 100644 --- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java @@ -42,8 +42,8 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.settings.Settings; @@ -53,7 +53,8 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.Engine; diff --git a/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java index a6ecb7053f286..040c47fae2858 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java @@ -52,6 +52,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; diff --git a/server/src/test/java/org/opensearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/opensearch/index/shard/ShardGetServiceTests.java index f29303ce8dda1..34d6233c8202f 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardGetServiceTests.java @@ -35,6 +35,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.VersionType; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.VersionConflictEngineException; diff --git a/server/src/test/java/org/opensearch/index/shard/ShardIdTests.java b/server/src/test/java/org/opensearch/index/shard/ShardIdTests.java index 6c5ae98bf6074..b124fd119e7e6 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardIdTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardIdTests.java @@ -33,7 +33,8 @@ package org.opensearch.index.shard; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; public class ShardIdTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/index/shard/ShardPathTests.java b/server/src/test/java/org/opensearch/index/shard/ShardPathTests.java index e445d4b78de9d..5bb74d7334113 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardPathTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardPathTests.java @@ -33,9 +33,10 @@ import org.opensearch.cluster.routing.AllocationId; import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.NodeEnvironment; import org.opensearch.gateway.WriteStateException; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java b/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java index fa30fbbac9a0d..14c925939ba9d 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java @@ -40,6 +40,7 @@ import org.apache.lucene.tests.store.BaseDirectoryWrapper; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java b/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java index 1f384285bd633..c1a51bb780f61 100644 --- a/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java @@ -57,6 +57,7 @@ import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.engine.Engine; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.Uid; diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java index 3b4db9e7a5a3c..0f957517d48e1 100644 --- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java +++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java index 1cb6579c0f1f2..ccb89ec3d1547 100644 --- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java +++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.snapshots.blobstore; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/server/src/test/java/org/opensearch/index/store/FsDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/FsDirectoryFactoryTests.java index ce40de0e9aa71..56d67820797a2 100644 --- a/server/src/test/java/org/opensearch/index/store/FsDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/FsDirectoryFactoryTests.java @@ -42,10 +42,10 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/index/store/RemoteBufferedIndexOutputTests.java b/server/src/test/java/org/opensearch/index/store/RemoteBufferedIndexOutputTests.java index b9200d71ac226..e78f18d764746 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteBufferedIndexOutputTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteBufferedIndexOutputTests.java @@ -13,9 +13,9 @@ import org.junit.After; import org.junit.Before; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index bf4b2a14f2567..ef23ce927eff9 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -20,7 +20,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index ea092fffa3a9a..72d8c234d71c8 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -29,12 +29,12 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; import org.opensearch.common.blobstore.stream.write.WriteContext; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/test/java/org/opensearch/index/store/StoreTests.java b/server/src/test/java/org/opensearch/index/store/StoreTests.java index d9493e63fd34c..9043dcce1b779 100644 --- a/server/src/test/java/org/opensearch/index/store/StoreTests.java +++ b/server/src/test/java/org/opensearch/index/store/StoreTests.java @@ -69,22 +69,22 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.ShardLock; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata; import org.opensearch.test.DummyShardLock; diff --git a/server/src/test/java/org/opensearch/index/store/StoreUtils.java b/server/src/test/java/org/opensearch/index/store/StoreUtils.java index 293d043c2d5f7..757ae6825c0bf 100644 --- a/server/src/test/java/org/opensearch/index/store/StoreUtils.java +++ b/server/src/test/java/org/opensearch/index/store/StoreUtils.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import java.nio.file.Path; diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java index 5bdba0513af23..eec9666cc06ba 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java @@ -25,7 +25,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java index dbcc646aaf638..7931c6fec5414 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.store.remote.filecache; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.store.remote.utils.cache.CacheUsage; import org.opensearch.index.store.remote.utils.cache.stats.CacheStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java b/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java index cc0764a6700b1..dc7940529f024 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java @@ -16,7 +16,7 @@ import org.junit.Before; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/index/suggest/stats/CompletionsStatsTests.java b/server/src/test/java/org/opensearch/index/suggest/stats/CompletionsStatsTests.java index 6d193bd9af4ac..104f132fb47ce 100644 --- a/server/src/test/java/org/opensearch/index/suggest/stats/CompletionsStatsTests.java +++ b/server/src/test/java/org/opensearch/index/suggest/stats/CompletionsStatsTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.FieldMemoryStatsTests; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.suggest.completion.CompletionStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java index a7d7b3a51cebb..788391f0e42c0 100644 --- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java @@ -55,13 +55,13 @@ import org.opensearch.common.Randomness; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; @@ -86,7 +86,7 @@ import org.opensearch.index.seqno.LocalCheckpointTracker; import org.opensearch.index.seqno.LocalCheckpointTrackerTests; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog.Location; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index 04dc90cd9087d..74b1fab30d83f 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -26,7 +26,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -44,7 +44,7 @@ import org.opensearch.index.seqno.LocalCheckpointTracker; import org.opensearch.index.seqno.LocalCheckpointTrackerTests; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.repositories.blobstore.BlobStoreRepository; diff --git a/server/src/test/java/org/opensearch/index/translog/TestTranslog.java b/server/src/test/java/org/opensearch/index/translog/TestTranslog.java index 73a3a1c646d5b..7a8ff88079200 100644 --- a/server/src/test/java/org/opensearch/index/translog/TestTranslog.java +++ b/server/src/test/java/org/opensearch/index/translog/TestTranslog.java @@ -36,7 +36,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.logging.log4j.Logger; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java index 86ac0c9405eae..d086732343bd2 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.mockito.Mockito; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java index 10b20ab207927..4441e30ea639d 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogHeaderTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java index 44e8d52b034de..91694060617ab 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java @@ -19,13 +19,13 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.AllocationId; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfig; @@ -37,7 +37,7 @@ import org.opensearch.index.mapper.Uid; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/index/translog/listener/TranslogListenerTests.java b/server/src/test/java/org/opensearch/index/translog/listener/TranslogListenerTests.java index 3a27d26a630d6..68dc63aa4b52e 100644 --- a/server/src/test/java/org/opensearch/index/translog/listener/TranslogListenerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/listener/TranslogListenerTests.java @@ -8,8 +8,8 @@ package org.opensearch.index.translog.listener; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.lang.reflect.Proxy; diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java index fd0d44564ef6b..1914790ac58d2 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/FileTransferTrackerTests.java @@ -8,7 +8,7 @@ package org.opensearch.index.translog.transfer; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 66cd257299e25..b7091f3f4f8a6 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -18,8 +18,8 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.blobstore.stream.write.WritePriority; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.transfer.FileSnapshot.CheckpointFileSnapshot; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferMetadataHandlerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferMetadataHandlerTests.java index ccedd4a711433..9b27e8ee8f3f5 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferMetadataHandlerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferMetadataHandlerTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.OutputStreamIndexOutput; import org.junit.Before; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java index 213a22539971f..c9bb263bf785e 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -39,14 +39,14 @@ import org.opensearch.cluster.routing.ShardRoutingHelper; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; diff --git a/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java index 5d293e6c598f6..5590c529cc9fd 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.settings.Settings; import org.opensearch.index.cache.query.QueryCacheStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index d10b99f42e0df..3508020cb23d7 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -46,8 +46,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedSupplier; -import org.opensearch.common.bytes.AbstractBytesReference; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.AbstractBytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; @@ -57,7 +57,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.cache.request.ShardRequestCache; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java index e3b2afaccf054..280a9ff2acf21 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java @@ -36,8 +36,8 @@ import org.apache.lucene.search.Query; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.routing.allocation.DiskThresholdSettings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.cache.RemovalNotification; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java index 7c2a87fd78847..2c0f061f13e58 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java @@ -54,7 +54,7 @@ import org.opensearch.gateway.GatewayMetaState; import org.opensearch.gateway.LocalAllocateDangledIndices; import org.opensearch.gateway.MetaStateService; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; @@ -69,7 +69,7 @@ import org.opensearch.index.shard.IllegalIndexShardStateException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.NonNegativeScoresSimilarity; import org.opensearch.indices.IndicesService.ShardDeletionCheckResult; diff --git a/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java b/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java index 8d0c8ed4690c4..6f3f6180e6ae0 100644 --- a/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java +++ b/server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java @@ -44,7 +44,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; diff --git a/server/src/test/java/org/opensearch/indices/TermsLookupTests.java b/server/src/test/java/org/opensearch/indices/TermsLookupTests.java index 898bc4e13327d..b37b6eb219950 100644 --- a/server/src/test/java/org/opensearch/indices/TermsLookupTests.java +++ b/server/src/test/java/org/opensearch/indices/TermsLookupTests.java @@ -33,7 +33,7 @@ package org.opensearch.indices; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java index c8e0460758df1..65a70b9f059f7 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java +++ b/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -43,7 +43,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; @@ -52,7 +52,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndex; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices; diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 14aabb49351b1..16f046306dc7c 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -100,7 +100,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.IndexEventListener; diff --git a/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index be64d89130950..6b1e4c8ebbed6 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -61,10 +61,10 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.shard.PrimaryReplicaSyncer; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.replication.SegmentReplicationSourceService; import org.opensearch.indices.replication.SegmentReplicationTargetService; diff --git a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java index 5654dad580ab6..0b2de2acbb1d0 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java @@ -59,8 +59,8 @@ import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.common.lucene.store.IndexOutputOutputStream; @@ -90,7 +90,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardRelocatedException; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.translog.Translog; diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java index 9faa8ddff8183..67811e24b03c4 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.routing.ShardRoutingHelper; import org.opensearch.common.Randomness; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTargetTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTargetTests.java index bb296f32ca8bd..7b171af4570d9 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTargetTests.java @@ -38,9 +38,9 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.common.ReplicationLuceneIndex.FileMetadata; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.recovery.RecoveryState.Stage; diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java index 97772564acc88..2c45c9e177c52 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/indices/recovery/StartRecoveryRequestTests.java b/server/src/test/java/org/opensearch/indices/recovery/StartRecoveryRequestTests.java index 9ad8403cea14d..a944c0c2de8ed 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/StartRecoveryRequestTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/StartRecoveryRequestTests.java @@ -35,11 +35,11 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.index.engine.Engine; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java index 1c78746a877e0..3b289114f5ca1 100644 --- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java @@ -21,7 +21,7 @@ import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.FileChunkWriter; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java index 8fc7b8b6f4908..d0065bfe013bd 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java @@ -14,7 +14,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java index 1ceba05f16369..bee1bedc892d8 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java @@ -15,14 +15,14 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ReplicationGroup; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 32203c28c8ed8..4643615d45d7e 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -22,11 +22,11 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.CancellableThreads; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.ForceSyncRequest; diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index 4fb1edb4e496e..176954b6d6b3d 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -36,7 +36,7 @@ import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.store.StoreTests; diff --git a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java index c851edf5e1bc8..48427ab7e5cd4 100644 --- a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java @@ -19,10 +19,10 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationTargetService; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java b/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java index 4f1b246f96231..13ab40203ff2b 100644 --- a/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java @@ -20,7 +20,7 @@ import org.opensearch.index.codec.CodecService; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; diff --git a/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java index 2e244908dc4eb..7e79551605a6e 100644 --- a/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java +++ b/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java @@ -48,8 +48,8 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.plugins.ActionPlugin; diff --git a/server/src/test/java/org/opensearch/indices/store/IndicesStoreTests.java b/server/src/test/java/org/opensearch/indices/store/IndicesStoreTests.java index d50145b08ee3d..79434e1b23e9d 100644 --- a/server/src/test/java/org/opensearch/indices/store/IndicesStoreTests.java +++ b/server/src/test/java/org/opensearch/indices/store/IndicesStoreTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java b/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java index f1d48f114035f..fdaf6d145da64 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java @@ -34,8 +34,8 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java index 19fef468c529e..7f5ec52c3620c 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java @@ -57,10 +57,10 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.cbor.CborXContent; diff --git a/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java b/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java index b17e24ee5424d..67b3d325b8a87 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestStatsTests.java @@ -34,8 +34,8 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.metrics.OperationStats; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java index 78350b7ab4c04..51ebd1b1f7ad5 100644 --- a/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java @@ -32,10 +32,10 @@ package org.opensearch.ingest; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/monitor/os/OsStatsTests.java b/server/src/test/java/org/opensearch/monitor/os/OsStatsTests.java index 66585962a8519..66f1c2c0cf981 100644 --- a/server/src/test/java/org/opensearch/monitor/os/OsStatsTests.java +++ b/server/src/test/java/org/opensearch/monitor/os/OsStatsTests.java @@ -33,7 +33,7 @@ package org.opensearch.monitor.os; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/node/InternalSettingsPreparerTests.java b/server/src/test/java/org/opensearch/node/InternalSettingsPreparerTests.java index 0f1aad5389c23..24560fb84878e 100644 --- a/server/src/test/java/org/opensearch/node/InternalSettingsPreparerTests.java +++ b/server/src/test/java/org/opensearch/node/InternalSettingsPreparerTests.java @@ -35,7 +35,7 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; diff --git a/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java b/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java index cdd1c682b40dc..7b17d4d3b0471 100644 --- a/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java +++ b/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/test/java/org/opensearch/persistent/CancelPersistentTaskRequestTests.java b/server/src/test/java/org/opensearch/persistent/CancelPersistentTaskRequestTests.java index 6fb61bc08729f..dae7a93e02596 100644 --- a/server/src/test/java/org/opensearch/persistent/CancelPersistentTaskRequestTests.java +++ b/server/src/test/java/org/opensearch/persistent/CancelPersistentTaskRequestTests.java @@ -31,7 +31,7 @@ package org.opensearch.persistent; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.persistent.RemovePersistentTaskAction.Request; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java b/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java index 98a070a9efa63..7fae4d5bb23e3 100644 --- a/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java +++ b/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java @@ -43,14 +43,14 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.core.ParseField; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/opensearch/persistent/PersistentTasksExecutorResponseTests.java index 22c2946fbcf40..6fad572bb4263 100644 --- a/server/src/test/java/org/opensearch/persistent/PersistentTasksExecutorResponseTests.java +++ b/server/src/test/java/org/opensearch/persistent/PersistentTasksExecutorResponseTests.java @@ -32,8 +32,8 @@ package org.opensearch.persistent; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.persistent.PersistentTasksCustomMetadata.PersistentTask; import org.opensearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/persistent/PersistentTasksNodeServiceStatusTests.java b/server/src/test/java/org/opensearch/persistent/PersistentTasksNodeServiceStatusTests.java index 9627fb96ff254..73c5d674a5816 100644 --- a/server/src/test/java/org/opensearch/persistent/PersistentTasksNodeServiceStatusTests.java +++ b/server/src/test/java/org/opensearch/persistent/PersistentTasksNodeServiceStatusTests.java @@ -31,7 +31,7 @@ package org.opensearch.persistent; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; import org.opensearch.persistent.PersistentTasksNodeService.Status; diff --git a/server/src/test/java/org/opensearch/persistent/RestartPersistentTaskRequestTests.java b/server/src/test/java/org/opensearch/persistent/RestartPersistentTaskRequestTests.java index 28503d0eba512..5adf8594cd983 100644 --- a/server/src/test/java/org/opensearch/persistent/RestartPersistentTaskRequestTests.java +++ b/server/src/test/java/org/opensearch/persistent/RestartPersistentTaskRequestTests.java @@ -31,7 +31,7 @@ package org.opensearch.persistent; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.persistent.CompletionPersistentTaskAction.Request; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/opensearch/persistent/StartPersistentActionRequestTests.java index fef2c88f0d61a..fe733525c6779 100644 --- a/server/src/test/java/org/opensearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/opensearch/persistent/StartPersistentActionRequestTests.java @@ -32,9 +32,9 @@ package org.opensearch.persistent; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.persistent.StartPersistentTaskAction.Request; import org.opensearch.persistent.TestPersistentTasksPlugin.TestParams; import org.opensearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; diff --git a/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java index 1d276c6a1c64e..3991f6721a350 100644 --- a/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java @@ -55,10 +55,10 @@ import org.opensearch.common.Strings; import org.opensearch.common.component.Lifecycle; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; diff --git a/server/src/test/java/org/opensearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/opensearch/persistent/UpdatePersistentTaskRequestTests.java index 06f426da34db5..e8a7ad1652139 100644 --- a/server/src/test/java/org/opensearch/persistent/UpdatePersistentTaskRequestTests.java +++ b/server/src/test/java/org/opensearch/persistent/UpdatePersistentTaskRequestTests.java @@ -32,8 +32,8 @@ package org.opensearch.persistent; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.persistent.TestPersistentTasksPlugin.State; import org.opensearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.opensearch.persistent.UpdatePersistentTaskStatusAction.Request; diff --git a/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java b/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java index 06295d34e9d97..2b2a8c7ae447c 100644 --- a/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java +++ b/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; -import org.opensearch.common.io.stream.ByteBufferStreamInput; +import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java b/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java index 0bb1bc210d946..776173f73ce5c 100644 --- a/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java +++ b/server/src/test/java/org/opensearch/recovery/ReplicationCollectionTests.java @@ -36,7 +36,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.indices.replication.SegmentReplicationSource; import org.opensearch.indices.replication.SegmentReplicationTarget; diff --git a/server/src/test/java/org/opensearch/repositories/IndexIdTests.java b/server/src/test/java/org/opensearch/repositories/IndexIdTests.java index 3554f2743698a..0e27dbded01bf 100644 --- a/server/src/test/java/org/opensearch/repositories/IndexIdTests.java +++ b/server/src/test/java/org/opensearch/repositories/IndexIdTests.java @@ -33,7 +33,7 @@ package org.opensearch.repositories; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index edf5b6c84bc54..6cbbb4ffcb7c7 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -56,7 +56,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.Store; diff --git a/server/src/test/java/org/opensearch/repositories/RepositoryDataTests.java b/server/src/test/java/org/opensearch/repositories/RepositoryDataTests.java index 8d76f4a91837d..26e4a2844a4ce 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoryDataTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoryDataTests.java @@ -35,7 +35,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 6797a1db23b2d..93958ce1ba7fb 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -54,8 +54,8 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; -import org.opensearch.index.snapshots.IndexShardSnapshotFailedException; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 105ccef500ce8..c750bf0f56e36 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -47,12 +47,12 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.RemoteBufferedOutputDirectory; import org.opensearch.indices.IndicesService; diff --git a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java index d7ba6f93e72c2..3049aa161f10b 100644 --- a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java @@ -63,7 +63,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.store.Store; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java index e504cf86ee32c..1f5e0cda883c5 100644 --- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java +++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java @@ -34,7 +34,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/rest/BytesRestResponseTests.java b/server/src/test/java/org/opensearch/rest/BytesRestResponseTests.java index a5d70886aa651..5bdc2cc0bd280 100644 --- a/server/src/test/java/org/opensearch/rest/BytesRestResponseTests.java +++ b/server/src/test/java/org/opensearch/rest/BytesRestResponseTests.java @@ -40,13 +40,14 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.ShardSearchFailure; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.transport.TransportAddress; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java index 489419fc24526..f8ea59040d22b 100644 --- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java @@ -34,8 +34,8 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -43,6 +43,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.yaml.YamlXContent; diff --git a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java index c1f5cd6ddf8bd..b181d571c14fb 100644 --- a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java +++ b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java @@ -33,10 +33,11 @@ package org.opensearch.rest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.rest.RestStatus; import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; diff --git a/server/src/test/java/org/opensearch/rest/RestRequestTests.java b/server/src/test/java/org/opensearch/rest/RestRequestTests.java index f5beb501e1681..15066362555ef 100644 --- a/server/src/test/java/org/opensearch/rest/RestRequestTests.java +++ b/server/src/test/java/org/opensearch/rest/RestRequestTests.java @@ -34,8 +34,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.MapBuilder; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/rest/action/RestActionListenerTests.java b/server/src/test/java/org/opensearch/rest/action/RestActionListenerTests.java index a674fff80e3f7..7c41607f12bfb 100644 --- a/server/src/test/java/org/opensearch/rest/action/RestActionListenerTests.java +++ b/server/src/test/java/org/opensearch/rest/action/RestActionListenerTests.java @@ -14,7 +14,7 @@ package org.opensearch.rest.action; import org.opensearch.OpenSearchException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/RestActionsTests.java b/server/src/test/java/org/opensearch/rest/action/RestActionsTests.java index cb3a6ddbf2e04..a49af7849dff8 100644 --- a/server/src/test/java/org/opensearch/rest/action/RestActionsTests.java +++ b/server/src/test/java/org/opensearch/rest/action/RestActionsTests.java @@ -34,10 +34,10 @@ import com.fasterxml.jackson.core.io.JsonEOFException; import org.opensearch.action.OriginalIndices; -import org.opensearch.action.ShardOperationFailedException; +import org.opensearch.core.action.ShardOperationFailedException; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -45,10 +45,10 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchModule; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/rest/action/RestBuilderListenerTests.java b/server/src/test/java/org/opensearch/rest/action/RestBuilderListenerTests.java index e93e8a932bd22..d865607aa5451 100644 --- a/server/src/test/java/org/opensearch/rest/action/RestBuilderListenerTests.java +++ b/server/src/test/java/org/opensearch/rest/action/RestBuilderListenerTests.java @@ -32,11 +32,11 @@ package org.opensearch.rest.action; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/RestMainActionTests.java b/server/src/test/java/org/opensearch/rest/action/RestMainActionTests.java index b128a3e30166f..e5e1001884e1e 100644 --- a/server/src/test/java/org/opensearch/rest/action/RestMainActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/RestMainActionTests.java @@ -36,13 +36,13 @@ import org.opensearch.Version; import org.opensearch.action.main.MainResponse; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterAddWeightedRoutingActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterAddWeightedRoutingActionTests.java index 582fbfce315b2..4d61ccad10b45 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterAddWeightedRoutingActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterAddWeightedRoutingActionTests.java @@ -12,7 +12,7 @@ import org.junit.Before; import org.opensearch.OpenSearchParseException; import org.opensearch.action.admin.cluster.shards.routing.weighted.put.ClusterPutWeightedRoutingRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterDeleteWeightedRoutingActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterDeleteWeightedRoutingActionTests.java index 2589d68e4cf0b..b11103a9cab11 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterDeleteWeightedRoutingActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterDeleteWeightedRoutingActionTests.java @@ -11,7 +11,7 @@ import org.junit.Before; import org.opensearch.OpenSearchParseException; import org.opensearch.action.admin.cluster.shards.routing.weighted.delete.ClusterDeleteWeightedRoutingRequest; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestAnalyzeActionTests.java index b9587f8d7c141..1c6c4eca6ca0d 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestAnalyzeActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestAnalyzeActionTests.java @@ -33,7 +33,7 @@ import org.opensearch.action.admin.indices.analyze.AnalyzeAction; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestCreateIndexActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestCreateIndexActionTests.java index 047fe0f286393..f6db8a0f24d4b 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestCreateIndexActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestCreateIndexActionTests.java @@ -32,7 +32,7 @@ package org.opensearch.rest.action.admin.indices; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java index 1b82787e92120..9739419406851 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -44,8 +44,8 @@ import java.util.List; import java.util.Map; -import static org.opensearch.rest.RestStatus.OK; -import static org.opensearch.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; import static org.hamcrest.Matchers.equalTo; public class RestGetAliasesActionTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java index e14b003e7ccad..094b4d8b8c12d 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -40,8 +40,8 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.TransportAction; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.identity.IdentityService; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java index 7a0d80d9538ad..44d8f84eef524 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java @@ -9,7 +9,7 @@ package org.opensearch.rest.action.cat; import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.ShardRouting; @@ -17,10 +17,10 @@ import org.opensearch.common.Table; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationShardStats; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationTimer; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 9c186e7fde314..fd74a9fe9e6cd 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -44,9 +44,9 @@ import org.opensearch.common.Table; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestRecoveryActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestRecoveryActionTests.java index efd31dda92d09..2f23209876ed0 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestRecoveryActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestRecoveryActionTests.java @@ -33,7 +33,7 @@ package org.opensearch.rest.action.cat; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; @@ -42,8 +42,8 @@ import org.opensearch.common.Table; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentOpenSearchExtension; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationTimer; diff --git a/server/src/test/java/org/opensearch/rest/action/document/RestBulkActionTests.java b/server/src/test/java/org/opensearch/rest/action/document/RestBulkActionTests.java index 1e13b365c82e3..f795d340778cf 100644 --- a/server/src/test/java/org/opensearch/rest/action/document/RestBulkActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/document/RestBulkActionTests.java @@ -39,7 +39,7 @@ import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; diff --git a/server/src/test/java/org/opensearch/rest/action/document/RestGetSourceActionTests.java b/server/src/test/java/org/opensearch/rest/action/document/RestGetSourceActionTests.java index ca6ecd052fe6a..2fc0391af593c 100644 --- a/server/src/test/java/org/opensearch/rest/action/document/RestGetSourceActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/document/RestGetSourceActionTests.java @@ -34,8 +34,8 @@ import org.opensearch.ResourceNotFoundException; import org.opensearch.action.get.GetResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.get.GetResult; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; @@ -48,7 +48,7 @@ import static java.util.Collections.emptyMap; import static org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; import static org.hamcrest.Matchers.equalTo; public class RestGetSourceActionTests extends RestActionTestCase { diff --git a/server/src/test/java/org/opensearch/rest/action/document/RestIndexActionTests.java b/server/src/test/java/org/opensearch/rest/action/document/RestIndexActionTests.java index fac1b2c75c9b7..4131e8d9a55c6 100644 --- a/server/src/test/java/org/opensearch/rest/action/document/RestIndexActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/document/RestIndexActionTests.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.document.RestIndexAction.AutoIdHandler; diff --git a/server/src/test/java/org/opensearch/rest/action/document/RestUpdateActionTests.java b/server/src/test/java/org/opensearch/rest/action/document/RestUpdateActionTests.java index bf544acf13508..f66f07a22e660 100644 --- a/server/src/test/java/org/opensearch/rest/action/document/RestUpdateActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/document/RestUpdateActionTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.rest.RestRequest; diff --git a/server/src/test/java/org/opensearch/script/ScriptCacheTests.java b/server/src/test/java/org/opensearch/script/ScriptCacheTests.java index 32e1c65cedbad..6395b8b315196 100644 --- a/server/src/test/java/org/opensearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptCacheTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/script/ScriptContextInfoTests.java b/server/src/test/java/org/opensearch/script/ScriptContextInfoTests.java index ebe00123a6d72..e10e199c4415c 100644 --- a/server/src/test/java/org/opensearch/script/ScriptContextInfoTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptContextInfoTests.java @@ -32,7 +32,7 @@ package org.opensearch.script; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/script/ScriptExceptionTests.java b/server/src/test/java/org/opensearch/script/ScriptExceptionTests.java index cda79234877ec..bc648ae4b46c7 100644 --- a/server/src/test/java/org/opensearch/script/ScriptExceptionTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptExceptionTests.java @@ -32,10 +32,10 @@ package org.opensearch.script; -import org.opensearch.common.io.stream.DataOutputStreamOutput; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.DataOutputStreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayInputStream; diff --git a/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java b/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java index 9c139f6de573f..bcab5dc7b776a 100644 --- a/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java @@ -32,9 +32,9 @@ package org.opensearch.script; import org.opensearch.cluster.DiffableUtils; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/script/ScriptServiceTests.java b/server/src/test/java/org/opensearch/script/ScriptServiceTests.java index 0485fc9935f15..489b267f586e4 100644 --- a/server/src/test/java/org/opensearch/script/ScriptServiceTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptServiceTests.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/script/ScriptTests.java b/server/src/test/java/org/opensearch/script/ScriptTests.java index 0740581e94342..47b4557c3e5e1 100644 --- a/server/src/test/java/org/opensearch/script/ScriptTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptTests.java @@ -34,8 +34,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/script/StoredScriptSourceTests.java b/server/src/test/java/org/opensearch/script/StoredScriptSourceTests.java index 1f33a51c9253c..548dccf5387c2 100644 --- a/server/src/test/java/org/opensearch/script/StoredScriptSourceTests.java +++ b/server/src/test/java/org/opensearch/script/StoredScriptSourceTests.java @@ -33,8 +33,8 @@ package org.opensearch.script; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/script/StoredScriptTests.java b/server/src/test/java/org/opensearch/script/StoredScriptTests.java index a943920dc1451..c16c4f93e6582 100644 --- a/server/src/test/java/org/opensearch/script/StoredScriptTests.java +++ b/server/src/test/java/org/opensearch/script/StoredScriptTests.java @@ -33,10 +33,10 @@ package org.opensearch.script; import org.opensearch.ResourceNotFoundException; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/opensearch/search/AbstractSearchTestCase.java index b487c0743c988..4a5e99006ac05 100644 --- a/server/src/test/java/org/opensearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/opensearch/search/AbstractSearchTestCase.java @@ -34,10 +34,10 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/ClearScrollResponseTests.java b/server/src/test/java/org/opensearch/search/ClearScrollResponseTests.java index c40af72097aee..27c127e2c9ea1 100644 --- a/server/src/test/java/org/opensearch/search/ClearScrollResponseTests.java +++ b/server/src/test/java/org/opensearch/search/ClearScrollResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.search; import org.opensearch.action.search.ClearScrollResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index 82ebae65a147b..8d8f2856d7703 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -64,7 +64,7 @@ import org.opensearch.index.query.ParsedQuery; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.LegacyReaderContext; diff --git a/server/src/test/java/org/opensearch/search/DeletePitResponseTests.java b/server/src/test/java/org/opensearch/search/DeletePitResponseTests.java index a45a37b33cfee..638231b470689 100644 --- a/server/src/test/java/org/opensearch/search/DeletePitResponseTests.java +++ b/server/src/test/java/org/opensearch/search/DeletePitResponseTests.java @@ -10,7 +10,7 @@ import org.opensearch.action.search.DeletePitInfo; import org.opensearch.action.search.DeletePitResponse; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java index 961b7bc3dec3f..18456441f1316 100644 --- a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java @@ -35,10 +35,10 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.time.DateFormatter; import org.opensearch.index.mapper.DateFieldMapper.Resolution; diff --git a/server/src/test/java/org/opensearch/search/MultiValueModeTests.java b/server/src/test/java/org/opensearch/search/MultiValueModeTests.java index eeb743848b600..948d2cffceabe 100644 --- a/server/src/test/java/org/opensearch/search/MultiValueModeTests.java +++ b/server/src/test/java/org/opensearch/search/MultiValueModeTests.java @@ -42,7 +42,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.fielddata.AbstractBinaryDocValues; import org.opensearch.index.fielddata.AbstractNumericDocValues; import org.opensearch.index.fielddata.AbstractSortedDocValues; diff --git a/server/src/test/java/org/opensearch/search/NestedIdentityTests.java b/server/src/test/java/org/opensearch/search/NestedIdentityTests.java index 688d7096dc670..341570247ffa9 100644 --- a/server/src/test/java/org/opensearch/search/NestedIdentityTests.java +++ b/server/src/test/java/org/opensearch/search/NestedIdentityTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/search/SearchHitTests.java b/server/src/test/java/org/opensearch/search/SearchHitTests.java index a921690d45c1a..04aa60b2be913 100644 --- a/server/src/test/java/org/opensearch/search/SearchHitTests.java +++ b/server/src/test/java/org/opensearch/search/SearchHitTests.java @@ -37,18 +37,18 @@ import org.opensearch.Version; import org.opensearch.action.OriginalIndices; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.get.GetResultTests; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit.NestedIdentity; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.search.fetch.subphase.highlight.HighlightFieldTests; diff --git a/server/src/test/java/org/opensearch/search/SearchHitsTests.java b/server/src/test/java/org/opensearch/search/SearchHitsTests.java index be8351e030c48..d5cee765edee8 100644 --- a/server/src/test/java/org/opensearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/opensearch/search/SearchHitsTests.java @@ -37,8 +37,8 @@ import org.apache.lucene.tests.util.TestUtil; import org.opensearch.action.OriginalIndices; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.LuceneTests; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.ToXContent; @@ -46,8 +46,8 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.AbstractSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/search/SearchModuleTests.java b/server/src/test/java/org/opensearch/search/SearchModuleTests.java index c0351c9dccbc1..375bceb16d03a 100644 --- a/server/src/test/java/org/opensearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/opensearch/search/SearchModuleTests.java @@ -33,8 +33,8 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.CharsRefBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.LoggingDeprecationHandler; diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java index 74d1df94214ee..2371c5812814a 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java @@ -55,8 +55,8 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.WriteRequest; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; @@ -64,7 +64,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; @@ -80,12 +80,12 @@ import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.SearchOperationListener; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.settings.InternalOrPrivateSettingsPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; diff --git a/server/src/test/java/org/opensearch/search/SearchSortValuesAndFormatsTests.java b/server/src/test/java/org/opensearch/search/SearchSortValuesAndFormatsTests.java index bc6648d693d1d..0266f5cfcce60 100644 --- a/server/src/test/java/org/opensearch/search/SearchSortValuesAndFormatsTests.java +++ b/server/src/test/java/org/opensearch/search/SearchSortValuesAndFormatsTests.java @@ -33,8 +33,8 @@ package org.opensearch.search; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.test.AbstractWireSerializingTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/search/SearchSortValuesTests.java b/server/src/test/java/org/opensearch/search/SearchSortValuesTests.java index b8332f659246f..36d01fa4eb988 100644 --- a/server/src/test/java/org/opensearch/search/SearchSortValuesTests.java +++ b/server/src/test/java/org/opensearch/search/SearchSortValuesTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.LuceneTests; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java index e94b8ccf83c1f..fd82cd21e0fa6 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesBuilderTests.java index 6a7bf232bce77..32fd3fe2ec325 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesBuilderTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java index 6e83739ac8771..1f29a17deeaa8 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java @@ -32,10 +32,10 @@ package org.opensearch.search.aggregations; import org.opensearch.action.ActionListener; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java index 2d022be10fe29..354c635e06fab 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/InternalAggregationsTests.java @@ -34,9 +34,9 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchModule; diff --git a/server/src/test/java/org/opensearch/search/aggregations/InternalOrderTests.java b/server/src/test/java/org/opensearch/search/aggregations/InternalOrderTests.java index f91ef93d453b8..190f35b955853 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/InternalOrderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/InternalOrderTests.java @@ -31,7 +31,7 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.search.aggregations.InternalOrder.CompoundOrder; diff --git a/server/src/test/java/org/opensearch/search/aggregations/SubAggCollectionModeTests.java b/server/src/test/java/org/opensearch/search/aggregations/SubAggCollectionModeTests.java index 2d402f8f9bc7a..9b80f0ba6b6d7 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/SubAggCollectionModeTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/SubAggCollectionModeTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapperTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapperTests.java index 7679e77d35eff..e17dc371f9ee7 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapperTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapperTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java index 3d62f3a93e70b..67fe82c414198 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java @@ -32,9 +32,9 @@ package org.opensearch.search.aggregations.bucket.histogram; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/LongBoundsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/LongBoundsTests.java index e927488edd920..6eadc12330bba 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/LongBoundsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/LongBoundsTests.java @@ -34,9 +34,9 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java index 0df7ce14d28ea..b926a5a7895f2 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.bucket.range; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index debe4d343e874..ff0b74b1d9d28 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -53,7 +53,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.IdFieldMapper; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificanceHeuristicTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificanceHeuristicTests.java index aa23c00c29e5f..810be2d796740 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificanceHeuristicTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificanceHeuristicTests.java @@ -34,11 +34,11 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index cb47bf6cba6a9..4ce66e0495f83 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -56,7 +56,7 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.index.mapper.DocumentMapper; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java index 2608c2dee0a23..786ac01db6ab6 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -41,7 +41,7 @@ import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/PercentilesMethodTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/PercentilesMethodTests.java index 7d4a02c7a2c1f..6634840b9af45 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/PercentilesMethodTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/PercentilesMethodTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.aggregations.metrics; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/TopHitsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/TopHitsAggregatorTests.java index 7dc80ab6f4a7b..20d1cf4ffd904 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/TopHitsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/TopHitsAggregatorTests.java @@ -52,7 +52,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java index a5151f22ee339..9bc0aff862b84 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.metrics.weighted_avg; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/aggregations/pipeline/BucketHelpersTests.java b/server/src/test/java/org/opensearch/search/aggregations/pipeline/BucketHelpersTests.java index fcc20c71fedae..b6cf355c098e5 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/pipeline/BucketHelpersTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/pipeline/BucketHelpersTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.AggregationExecutionException; import org.opensearch.search.aggregations.Aggregations; diff --git a/server/src/test/java/org/opensearch/search/aggregations/pipeline/GapPolicyTests.java b/server/src/test/java/org/opensearch/search/aggregations/pipeline/GapPolicyTests.java index 0eb064079334c..6a3b7e4f9dcb0 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/pipeline/GapPolicyTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/pipeline/GapPolicyTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations.pipeline; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.io.stream.AbstractWriteableEnumTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfigTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfigTests.java index cf70f722cef9f..3b668f3c78c06 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfigTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfigTests.java @@ -8,8 +8,8 @@ package org.opensearch.search.aggregations.support; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java index 83208618859db..3f348e34f4c27 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.support; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index 1f766b285b3d8..f62ef8af22828 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -13,8 +13,8 @@ import org.opensearch.Version; import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.search.SearchTask; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.search.backpressure.settings.SearchBackpressureMode; diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchBackpressureStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchBackpressureStatsTests.java index 0c86cf4b11239..ffbfa054b4837 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchBackpressureStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchBackpressureStatsTests.java @@ -8,7 +8,7 @@ package org.opensearch.search.backpressure.stats; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.backpressure.settings.SearchBackpressureMode; import org.opensearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java index d5bc9398492eb..6478fdfff61d4 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java @@ -8,7 +8,7 @@ package org.opensearch.search.backpressure.stats; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java index 07cec723efb17..eb33bc1c37b7e 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java @@ -8,7 +8,7 @@ package org.opensearch.search.backpressure.stats; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java index 09c4fbc1ba251..461812077eba9 100644 --- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java @@ -33,11 +33,11 @@ package org.opensearch.search.builder; import com.fasterxml.jackson.core.JsonParseException; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/opensearch/search/collapse/CollapseBuilderTests.java index 5ae7f76d1de6b..afbde486f4b92 100644 --- a/server/src/test/java/org/opensearch/search/collapse/CollapseBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/collapse/CollapseBuilderTests.java @@ -38,8 +38,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/FetchSourcePhaseTests.java index 7356e64ed329b..882065866243a 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.memory.MemoryIndex; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java index f6d5789c1d503..447b4a0354637 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighter.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighter.java index 710e89670de0a..649c9e26a70f3 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighter.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighter.java @@ -31,7 +31,7 @@ package org.opensearch.search.fetch.subphase.highlight; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.index.mapper.MappedFieldType; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index eb3083b215f30..b1dfd836f7461 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -35,10 +35,10 @@ import org.apache.lucene.search.Query; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -49,7 +49,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.MappedFieldType; diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java index 7ce78aa35856e..a66c7de6ced74 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java @@ -34,8 +34,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index c9755cec2a33d..a2c4288e081b3 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -80,8 +80,8 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java index 6aa330948cdfb..1f632251cf588 100644 --- a/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java @@ -51,7 +51,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RandomQueryBuilder; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.InvalidAliasNameException; import org.opensearch.search.AbstractSearchTestCase; import org.opensearch.search.SearchSortValuesAndFormatsTests; diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java index 6eb137cb28e8f..c19c1ebcb5c26 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java @@ -10,7 +10,7 @@ import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index 0103d4c677b00..118d77a7a5c41 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -39,14 +39,14 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.NoopCircuitBreaker; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.index.IndexSettings; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.plugins.SearchPipelinePlugin; diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java index dac41f0db4e00..90a6e99057b0e 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineStatsTests.java @@ -8,13 +8,13 @@ package org.opensearch.search.pipeline; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java b/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java index c54cf5b5395ce..4880afb896a40 100644 --- a/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java @@ -13,7 +13,7 @@ import org.opensearch.action.search.DeletePitResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.search.RestDeletePitAction; diff --git a/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java b/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java index 121ae1646f5de..ae5a07478e814 100644 --- a/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.profile; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -50,7 +50,7 @@ import java.util.function.Predicate; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java b/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java index e677c97489c56..4c6853e598ffe 100644 --- a/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java +++ b/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.profile; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -50,8 +50,8 @@ import java.util.function.Predicate; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java b/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java index 8bc32efdad489..33c95725dcd13 100644 --- a/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java @@ -32,10 +32,10 @@ package org.opensearch.search.profile.aggregation; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.profile.ProfileResult; import org.opensearch.search.profile.ProfileResultTests; diff --git a/server/src/test/java/org/opensearch/search/profile/query/CollectorResultTests.java b/server/src/test/java/org/opensearch/search/profile/query/CollectorResultTests.java index 6cdcf3a9690c9..a3ac26a28ab1f 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/CollectorResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/CollectorResultTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.profile.query; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -47,7 +47,7 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java b/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java index a878acc8868c3..e703396f5cf02 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java @@ -32,10 +32,10 @@ package org.opensearch.search.profile.query; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.profile.ProfileResult; import org.opensearch.search.profile.ProfileResultTests; diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index 6347d5e722581..912929db10a5b 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -40,11 +40,11 @@ import org.opensearch.action.OriginalIndicesTests; import org.opensearch.action.search.SearchRequest; import org.opensearch.common.UUIDs; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchModule; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java index 508bd210431fb..aa7bcf4d63e3d 100644 --- a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java @@ -36,8 +36,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedObjectNotFoundException; diff --git a/server/src/test/java/org/opensearch/search/scroll/RestClearScrollActionTests.java b/server/src/test/java/org/opensearch/search/scroll/RestClearScrollActionTests.java index a4605521634fc..8625c0004c66e 100644 --- a/server/src/test/java/org/opensearch/search/scroll/RestClearScrollActionTests.java +++ b/server/src/test/java/org/opensearch/search/scroll/RestClearScrollActionTests.java @@ -37,7 +37,7 @@ import org.opensearch.action.search.ClearScrollResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.search.RestClearScrollAction; diff --git a/server/src/test/java/org/opensearch/search/scroll/RestSearchScrollActionTests.java b/server/src/test/java/org/opensearch/search/scroll/RestSearchScrollActionTests.java index 7db3714769bcc..591f9e5564467 100644 --- a/server/src/test/java/org/opensearch/search/scroll/RestSearchScrollActionTests.java +++ b/server/src/test/java/org/opensearch/search/scroll/RestSearchScrollActionTests.java @@ -37,7 +37,7 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.common.SetOnce; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.search.RestSearchScrollAction; diff --git a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java index ca4cb7291b862..47579e31dc7c2 100644 --- a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java @@ -37,10 +37,10 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.text.Text; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java index 692080b592502..28d96c916bc41 100644 --- a/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java @@ -54,7 +54,7 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; @@ -67,7 +67,7 @@ import org.opensearch.index.mapper.TextSearchInfo; import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.lookup.SearchLookup; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java index 5fa68ca7e940f..e8d713d15b337 100644 --- a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java @@ -37,7 +37,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.TriFunction; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -47,7 +47,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.env.Environment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.fielddata.IndexFieldData; diff --git a/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java index d632d627024ef..bf37e822bfc3c 100644 --- a/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.sort; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/search/sort/SortOrderTests.java b/server/src/test/java/org/opensearch/search/sort/SortOrderTests.java index d2c4d248c22ee..a7cec3d12bcaf 100644 --- a/server/src/test/java/org/opensearch/search/sort/SortOrderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/SortOrderTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.sort; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/search/sort/SortValueTests.java b/server/src/test/java/org/opensearch/search/sort/SortValueTests.java index f35c6b553d974..e11a8064f7a85 100644 --- a/server/src/test/java/org/opensearch/search/sort/SortValueTests.java +++ b/server/src/test/java/org/opensearch/search/sort/SortValueTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.sort; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java b/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java index 38f3b73bf5755..8c5736fefb9ba 100644 --- a/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java +++ b/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java @@ -11,8 +11,8 @@ import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 1bdcf3cd9bf08..eee13ec56d841 100644 --- a/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -35,8 +35,8 @@ import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -45,7 +45,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.NamedAnalyzer; diff --git a/server/src/test/java/org/opensearch/search/suggest/CompletionSuggestionOptionTests.java b/server/src/test/java/org/opensearch/search/suggest/CompletionSuggestionOptionTests.java index 0dce9968a1a9f..d312299b924af 100644 --- a/server/src/test/java/org/opensearch/search/suggest/CompletionSuggestionOptionTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/CompletionSuggestionOptionTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.suggest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java index a8500bb152b00..f8996c528cf32 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.suggest; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestTests.java index 636593f9a7d80..e046b415dbcf3 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestTests.java @@ -34,13 +34,13 @@ import org.opensearch.Version; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -67,8 +67,8 @@ import static java.util.Collections.emptyList; import static org.opensearch.common.xcontent.XContentHelper.stripWhitespace; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestionEntryTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestionEntryTests.java index 6a1e3feb514ac..f8cc6bb5c9f2e 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestionEntryTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestionEntryTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.suggest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -53,7 +53,7 @@ import java.util.function.Supplier; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestionOptionTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestionOptionTests.java index d4b22d92c2a76..76d8301d6260d 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestionOptionTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestionOptionTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.suggest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -44,7 +44,7 @@ import java.io.IOException; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestionTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestionTests.java index a221b53ccf027..f1933ca43f2f9 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestionTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestionTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.suggest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -59,7 +59,7 @@ import java.util.function.Supplier; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/suggest/TermSuggestionOptionTests.java b/server/src/test/java/org/opensearch/search/suggest/TermSuggestionOptionTests.java index 549c4cf083746..b032f7729cb1d 100644 --- a/server/src/test/java/org/opensearch/search/suggest/TermSuggestionOptionTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/TermSuggestionOptionTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.suggest; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -43,7 +43,7 @@ import java.io.IOException; import static org.opensearch.common.xcontent.XContentHelper.toXContent; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.test.XContentTestUtils.insertRandomFields; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java index 248c5280ae54c..478e99c269f37 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java @@ -42,7 +42,7 @@ import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggesterBuilderTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggesterBuilderTests.java index d03b8e8edf763..868e60f049f68 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggesterBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggesterBuilderTests.java @@ -33,7 +33,7 @@ package org.opensearch.search.suggest.completion; import org.apache.lucene.analysis.core.SimpleAnalyzer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.index.analysis.AnalyzerScope; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggestionTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggestionTests.java index a618fffea3368..8398ec696c43c 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggestionTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/CompletionSuggestionTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.suggest.completion; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.search.suggest.Suggest; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/FuzzyOptionsTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/FuzzyOptionsTests.java index bc4f4e029e408..7856c106ac24f 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/FuzzyOptionsTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/FuzzyOptionsTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.suggest.completion; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.unit.Fuzziness; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java index fb6cce3f8e9ab..d641090f1a5aa 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java @@ -34,7 +34,7 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/GeoQueryContextTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/GeoQueryContextTests.java index 377601c5ed435..654bea6fd9b6c 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/GeoQueryContextTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/GeoQueryContextTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.suggest.completion; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.geo.GeoPoint; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/RegexOptionsTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/RegexOptionsTests.java index 57bc0e5708f4d..9d158f57703b6 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/RegexOptionsTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/RegexOptionsTests.java @@ -32,7 +32,7 @@ package org.opensearch.search.suggest.completion; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.index.query.RegexpFlag; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java index 88b523c665ffb..8484861999c3f 100644 --- a/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java @@ -46,7 +46,7 @@ import org.apache.lucene.search.spell.SuggestMode; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java b/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java index d47dc007274f5..a4adf03cff316 100644 --- a/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -43,7 +43,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiTerms; import org.apache.lucene.store.ByteBuffersDirectory; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 1c43fe73c8a78..3129dce795cf8 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -39,13 +39,13 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.fs.FsBlobStore; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.Compressor; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.compress.CompressorType; import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java index 769a3db0481cd..8cfc11cb6f32c 100644 --- a/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -55,8 +55,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.IndexId; diff --git a/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java index 51699734038ff..3f71a9da8354f 100644 --- a/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/RepositoriesMetadataSerializationTests.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractDiffableSerializationTestCase; diff --git a/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java index e607e283f4da1..b56f1b47edc33 100644 --- a/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.metadata.DataStream; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotInfoTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotInfoTests.java index 9a70ec58697d9..850a392c9619c 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotInfoTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotInfoTests.java @@ -32,8 +32,8 @@ package org.opensearch.snapshots; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.AbstractWireSerializingTestCase; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java index b18aa6c53c107..a1acb75d9dd45 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java @@ -35,7 +35,7 @@ import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 88899a1b282af..e461e457bead1 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -150,7 +150,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; @@ -167,7 +167,7 @@ import org.opensearch.env.TestEnvironment; import org.opensearch.gateway.MetaStateService; import org.opensearch.gateway.TransportNodesListGatewayStartedShards; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.analysis.AnalysisRegistry; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotShardsServiceTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotShardsServiceTests.java index 39f13ecbbbee3..63af67d96974b 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotShardsServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotShardsServiceTests.java @@ -33,7 +33,7 @@ import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.UUIDs; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java index 417bc4d0421a0..e388732dbb7e2 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index bf491cacc88d1..8fd1f44286094 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -42,11 +42,11 @@ import org.opensearch.cluster.SnapshotsInProgress.State; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.test.AbstractDiffableWireSerializationTestCase; import org.opensearch.test.VersionUtils; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java index 9b12212c791a2..1652ba29b8fa7 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoryShardId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java b/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java index f85891b8091cf..5756397adce8a 100644 --- a/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java +++ b/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java @@ -41,7 +41,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.support.PlainBlobMetadata; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.Streams; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/server/src/test/java/org/opensearch/tasks/SearchShardTaskCancellationStatsTests.java b/server/src/test/java/org/opensearch/tasks/SearchShardTaskCancellationStatsTests.java index 4bab365536a49..ab265b613b897 100644 --- a/server/src/test/java/org/opensearch/tasks/SearchShardTaskCancellationStatsTests.java +++ b/server/src/test/java/org/opensearch/tasks/SearchShardTaskCancellationStatsTests.java @@ -8,7 +8,7 @@ package org.opensearch.tasks; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; public class SearchShardTaskCancellationStatsTests extends AbstractWireSerializingTestCase { diff --git a/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java index e068e6ee6e319..56e851871187b 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java @@ -12,7 +12,7 @@ import org.junit.Before; import org.opensearch.Version; import org.opensearch.action.search.SearchShardTask; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java b/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java index a81110b59e98a..beef05b04ab7a 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java @@ -8,7 +8,7 @@ package org.opensearch.tasks; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; public class TaskCancellationStatsTests extends AbstractWireSerializingTestCase { diff --git a/server/src/test/java/org/opensearch/tasks/TaskIdTests.java b/server/src/test/java/org/opensearch/tasks/TaskIdTests.java index 2789daa7f0481..0e77f7523cea0 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskIdTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskIdTests.java @@ -33,9 +33,9 @@ package org.opensearch.tasks; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/tasks/TaskInfoTests.java b/server/src/test/java/org/opensearch/tasks/TaskInfoTests.java index 0ba8ea13cc71c..6ee91cec29d49 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskInfoTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskInfoTests.java @@ -33,9 +33,9 @@ package org.opensearch.tasks; import org.opensearch.client.Requests; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/tasks/TaskResultTests.java b/server/src/test/java/org/opensearch/tasks/TaskResultTests.java index 758b206209e78..f27a920d812e9 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResultTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResultTests.java @@ -33,9 +33,9 @@ package org.opensearch.tasks; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/threadpool/ThreadPoolSerializationTests.java b/server/src/test/java/org/opensearch/threadpool/ThreadPoolSerializationTests.java index f268b80edb44c..935c8f6910c21 100644 --- a/server/src/test/java/org/opensearch/threadpool/ThreadPoolSerializationTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ThreadPoolSerializationTests.java @@ -32,9 +32,9 @@ package org.opensearch.threadpool; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.SizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java b/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java index 1c300bc5e2adf..b9071d5851315 100644 --- a/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java +++ b/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java @@ -32,12 +32,12 @@ package org.opensearch.transport; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; -import org.opensearch.common.io.stream.BytesStream; +import org.opensearch.core.common.io.stream.BytesStream; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.EOFException; diff --git a/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java b/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java index e9ffa4d478b22..dfb14be54a5b9 100644 --- a/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.breaker.TestCircuitBreaker; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java b/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java index c156a2d1570f8..4d8955650f8be 100644 --- a/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index c340042f99d86..d4959472bd08a 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -38,14 +38,14 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java index 12742c3228bfd..33fe928d5c09d 100644 --- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java @@ -37,8 +37,8 @@ import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.breaker.NoopCircuitBreaker; import org.opensearch.common.breaker.TestCircuitBreaker; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index 1cc115bf65a00..c61ac51ba57d4 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.NoopCircuitBreaker; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java index 74084757e3dd7..4106ebd9988c9 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java index 5190495a6253e..d1d46d5be13c0 100644 --- a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java @@ -35,8 +35,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/transport/TransportDecompressorTests.java b/server/src/test/java/org/opensearch/transport/TransportDecompressorTests.java index d5cd6dc49be12..35caedae00edb 100644 --- a/server/src/test/java/org/opensearch/transport/TransportDecompressorTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportDecompressorTests.java @@ -32,15 +32,15 @@ package org.opensearch.transport; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.lease.Releasables; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/transport/TransportHandshakerTests.java b/server/src/test/java/org/opensearch/transport/TransportHandshakerTests.java index ae2f6f884ac3e..58c8806380436 100644 --- a/server/src/test/java/org/opensearch/transport/TransportHandshakerTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportHandshakerTests.java @@ -35,7 +35,7 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.tasks.TaskId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/transport/TransportKeepAliveTests.java b/server/src/test/java/org/opensearch/transport/TransportKeepAliveTests.java index 924826d1a7f07..080b333251db2 100644 --- a/server/src/test/java/org/opensearch/transport/TransportKeepAliveTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportKeepAliveTests.java @@ -32,7 +32,7 @@ package org.opensearch.transport; import org.opensearch.common.AsyncBiFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java b/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java index 26f77d2fff9ee..fb0c29a82ac42 100644 --- a/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java @@ -36,7 +36,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.cluster.stats.ClusterStatsAction; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java index 8e11291e691e1..01d6a4d331477 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.coordination.DeterministicTaskQueue; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskAwareRequest; diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java index 357d7fffa8df6..71c69acb75896 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; diff --git a/test/external-modules/delayed-aggs/src/main/java/org/opensearch/search/aggregations/DelayedShardAggregationBuilder.java b/test/external-modules/delayed-aggs/src/main/java/org/opensearch/search/aggregations/DelayedShardAggregationBuilder.java index 77930f8143df5..773c969e2116a 100644 --- a/test/external-modules/delayed-aggs/src/main/java/org/opensearch/search/aggregations/DelayedShardAggregationBuilder.java +++ b/test/external-modules/delayed-aggs/src/main/java/org/opensearch/search/aggregations/DelayedShardAggregationBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.search.aggregations; import org.opensearch.core.ParseField; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java index 4879425b7bcd6..f3589dcfce3e5 100644 --- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java +++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java @@ -35,11 +35,11 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.Streams; import org.opensearch.common.regex.Regex; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import java.io.ByteArrayOutputStream; diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java index 611ec7b5198d9..f4ef1be4a9a3d 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java @@ -34,7 +34,7 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import org.opensearch.common.SuppressForbidden; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java index a3741668465fa..358d573629e0a 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java @@ -38,14 +38,14 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; import org.opensearch.common.regex.Regex; import org.opensearch.core.common.Strings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import java.io.BufferedReader; diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java index 88bd6a63c270d..d845b0b2a1b19 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java @@ -34,7 +34,7 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.net.InetAddress; diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java index 882d24145d67c..9e02f9ee86744 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java @@ -32,7 +32,7 @@ package fixture.s3; import com.sun.net.httpserver.HttpHandler; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.nio.charset.StandardCharsets; import java.time.ZonedDateTime; diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java index 7e261717f8201..ac5e3fdfdf85c 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java @@ -32,7 +32,7 @@ package fixture.s3; import com.sun.net.httpserver.HttpHandler; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.nio.charset.StandardCharsets; import java.util.Objects; diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java index b26c82a3cb7d4..8a28656760bcc 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java @@ -32,7 +32,7 @@ package fixture.s3; import com.sun.net.httpserver.HttpHandler; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java index b9a4fe36b687b..e5da27c639ca6 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java @@ -32,7 +32,7 @@ package fixture.s3; import com.sun.net.httpserver.HttpHandler; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.Objects; diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java index 2f9c7f7e256d9..3863cda9b8c96 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java @@ -37,13 +37,13 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.hash.MessageDigests; import org.opensearch.common.io.Streams; import org.opensearch.common.regex.Regex; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestUtils; import java.io.BufferedInputStream; diff --git a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java index 9dcb5a07030cf..182b2c9288a3d 100644 --- a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java @@ -50,7 +50,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.settings.Settings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/test/framework/src/main/java/org/opensearch/cluster/DataStreamTestHelper.java b/test/framework/src/main/java/org/opensearch/cluster/DataStreamTestHelper.java index af78fd36dceb4..c97260db43e80 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/opensearch/cluster/DataStreamTestHelper.java @@ -40,7 +40,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 1b42b2f70920f..98b4774ce1836 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -60,9 +60,9 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/test/framework/src/main/java/org/opensearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/opensearch/cluster/routing/TestShardRouting.java index 76325b6a0035b..062af17ce68ca 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/opensearch/cluster/routing/TestShardRouting.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; diff --git a/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java b/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java index dd71711f9154c..07fd383dc230a 100644 --- a/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java @@ -38,9 +38,12 @@ import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.AbstractBytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.ByteArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; diff --git a/test/framework/src/main/java/org/opensearch/common/settings/MockSecureSettings.java b/test/framework/src/main/java/org/opensearch/common/settings/MockSecureSettings.java index 6144e89deadb3..28660563a7ab7 100644 --- a/test/framework/src/main/java/org/opensearch/common/settings/MockSecureSettings.java +++ b/test/framework/src/main/java/org/opensearch/common/settings/MockSecureSettings.java @@ -33,6 +33,7 @@ package org.opensearch.common.settings; import org.opensearch.common.hash.MessageDigests; +import org.opensearch.core.common.settings.SecureString; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java index ed7a1643bf674..5ef68f0eab757 100644 --- a/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java @@ -40,6 +40,8 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.util.set.Sets; +import org.opensearch.core.common.util.BigArray; +import org.opensearch.core.common.util.ByteArray; import org.opensearch.indices.breaker.CircuitBreakerService; import java.util.Collection; diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index f7ec1faf6b9d8..9c6f36c87896c 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -73,8 +73,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Randomness; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lucene.Lucene; @@ -88,7 +88,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.MapperTestUtils; import org.opensearch.index.VersionType; @@ -109,7 +109,7 @@ import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.translog.InternalTranslogManager; import org.opensearch.index.translog.LocalTranslog; diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java index 30cbe5dcc6091..9d39b488070be 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java @@ -40,8 +40,8 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java index 420cb5f86e95f..57c09db577247 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java @@ -45,7 +45,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java index b815a819756b7..f3c98ce4f9f03 100644 --- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java @@ -74,14 +74,14 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.collect.Iterators; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.DocIdSeqNoAndSource; import org.opensearch.index.engine.EngineConfigFactory; @@ -96,7 +96,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.shard.PrimaryReplicaSyncer; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 984605007dad7..b23c28be14865 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -67,7 +67,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.io.PathUtils; import org.opensearch.common.lease.Releasable; @@ -78,8 +78,9 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.index.Index; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.MapperTestUtils; import org.opensearch.index.VersionType; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java index 67f3f7fc1f50c..a9e1a526b1786 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java @@ -41,6 +41,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.store.Store; diff --git a/test/framework/src/main/java/org/opensearch/indices/recovery/AsyncRecoveryTarget.java b/test/framework/src/main/java/org/opensearch/indices/recovery/AsyncRecoveryTarget.java index fa8b3c9e3a2c3..b3398fb752be5 100644 --- a/test/framework/src/main/java/org/opensearch/indices/recovery/AsyncRecoveryTarget.java +++ b/test/framework/src/main/java/org/opensearch/indices/recovery/AsyncRecoveryTarget.java @@ -34,8 +34,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.shard.IndexShard; diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index 1d8d2dd086418..0bd6e1872a101 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.MockInternalClusterInfoService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 0361652eb2457..bcbf3b5b9a610 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -37,7 +37,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; import org.opensearch.common.unit.ByteSizeValue; diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchBlobStoreRepositoryIntegTestCase.java index c3075ef7c8a67..3b7a921381882 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchBlobStoreRepositoryIntegTestCase.java @@ -46,7 +46,7 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.compress.CompressorType; import org.opensearch.common.io.Streams; import org.opensearch.common.settings.Settings; diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java index 31ba7bb89c60e..3fe46a3cb3c86 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java @@ -44,7 +44,7 @@ import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; diff --git a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java index 9dec1f2ba739c..ae993da34bdf5 100644 --- a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java @@ -35,8 +35,8 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchType; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.DeprecationHandler; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index cf0f9dbb04f7f..3cddf4659fea9 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -64,7 +64,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.TriFunction; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.settings.Settings; @@ -75,7 +75,7 @@ import org.opensearch.common.lease.Releasables; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.analysis.AnalyzerScope; @@ -108,7 +108,7 @@ import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.NoneCircuitBreakerService; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java index 54e084a0b662c..89d705c8fd1e8 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java @@ -34,8 +34,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/BasePipelineAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/BasePipelineAggregationTestCase.java index b6211b9ddbd64..fdbe7d73172af 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/BasePipelineAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/BasePipelineAggregationTestCase.java @@ -44,9 +44,9 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/InternalSingleBucketAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/InternalSingleBucketAggregationTestCase.java index 71625c498d063..589590c07e873 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/InternalSingleBucketAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/InternalSingleBucketAggregationTestCase.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.action.search.RestSearchAction; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java index 7d00772913d6e..66d167499b2cf 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java @@ -36,10 +36,10 @@ import org.junit.After; import org.junit.Before; import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.DocumentMapper; diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index de5a569bebaf9..21c3b30b24939 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -53,11 +53,11 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorType; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractBroadcastResponseTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractBroadcastResponseTestCase.java index 00b3bae7ba0c6..af33d5b66226e 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractBroadcastResponseTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractBroadcastResponseTestCase.java @@ -33,16 +33,16 @@ package org.opensearch.test; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.Index; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java index 99659e06102b3..527e04b6a4eb5 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java @@ -49,7 +49,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Strings; import org.opensearch.common.compress.CompressedXContent; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; @@ -60,7 +60,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.cache.bitset.BitsetFilterCache; @@ -69,7 +69,7 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.analysis.AnalysisModule; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java index 5f2bbce0a6861..f5d358a162bd1 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.Diffable; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.core.xcontent.ToXContent; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java index 4c692cc2bf726..ff7c39cd8f102 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java @@ -34,7 +34,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.Diffable; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.Writeable.Reader; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractNamedWriteableTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractNamedWriteableTestCase.java index b78160704992c..d1bdc621a92dd 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractNamedWriteableTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractNamedWriteableTestCase.java @@ -33,9 +33,9 @@ package org.opensearch.test; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java index 33d292ddb65cd..096b265d81019 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java @@ -41,14 +41,14 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable.Reader; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractSerializingTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractSerializingTestCase.java index 152691bdb38a3..f3496aabcb292 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractSerializingTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractSerializingTestCase.java @@ -32,7 +32,7 @@ package org.opensearch.test; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractWireSerializingTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractWireSerializingTestCase.java index 0ae8f5f2c2012..4091f9dc61884 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractWireSerializingTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractWireSerializingTestCase.java @@ -32,10 +32,10 @@ package org.opensearch.test; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java index ac5b48675481e..9a4363dc4d946 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java @@ -33,9 +33,9 @@ package org.opensearch.test; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; import java.util.Collections; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractXContentTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractXContentTestCase.java index d2dfa4ef628e5..a83399d902c22 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractXContentTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractXContentTestCase.java @@ -35,7 +35,7 @@ import org.opensearch.common.CheckedBiConsumer; import org.opensearch.common.CheckedBiFunction; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; diff --git a/test/framework/src/main/java/org/opensearch/test/DiffableTestUtils.java b/test/framework/src/main/java/org/opensearch/test/DiffableTestUtils.java index 50b9e02f84c21..25d3b1d9e6c5a 100644 --- a/test/framework/src/main/java/org/opensearch/test/DiffableTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/DiffableTestUtils.java @@ -35,11 +35,11 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.Diffable; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable.Reader; import java.io.IOException; import java.util.function.Function; diff --git a/test/framework/src/main/java/org/opensearch/test/DummyShardLock.java b/test/framework/src/main/java/org/opensearch/test/DummyShardLock.java index fbd2a03d6a9c2..82e5fa8733c42 100644 --- a/test/framework/src/main/java/org/opensearch/test/DummyShardLock.java +++ b/test/framework/src/main/java/org/opensearch/test/DummyShardLock.java @@ -33,7 +33,7 @@ package org.opensearch.test; import org.opensearch.env.ShardLock; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; /* * A ShardLock that does nothing... for tests only diff --git a/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java index e84bde1a064a8..4eb73487ac886 100644 --- a/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java @@ -42,7 +42,7 @@ import org.opensearch.client.Client; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/test/framework/src/main/java/org/opensearch/test/IndexSettingsModule.java b/test/framework/src/main/java/org/opensearch/test/IndexSettingsModule.java index b98b69a1c4201..2aea390fdc7f5 100644 --- a/test/framework/src/main/java/org/opensearch/test/IndexSettingsModule.java +++ b/test/framework/src/main/java/org/opensearch/test/IndexSettingsModule.java @@ -37,7 +37,7 @@ import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import java.util.Arrays; diff --git a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java index f335a18034e23..95246e06d028c 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java @@ -34,8 +34,8 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.MockBigArrays; @@ -45,7 +45,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.plugins.Plugin; diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 3f7bb71b27681..5134c8dd06fbb 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -70,7 +70,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.component.LifecycleListener; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.SecureSettings; import org.opensearch.common.settings.Settings; @@ -91,7 +91,7 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLockObtainFailedException; import org.opensearch.http.HttpServerTransport; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressure; import org.opensearch.index.engine.DocIdSeqNoAndSource; @@ -102,7 +102,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; diff --git a/test/framework/src/main/java/org/opensearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/opensearch/test/MockIndexEventListener.java index ecdae6a0e8b64..b79343c418508 100644 --- a/test/framework/src/main/java/org/opensearch/test/MockIndexEventListener.java +++ b/test/framework/src/main/java/org/opensearch/test/MockIndexEventListener.java @@ -37,14 +37,14 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.opensearch.plugins.Plugin; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index fec45219ace81..3564bd667ee2b 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -70,7 +70,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.ClearScrollResponse; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.AdminClient; import org.opensearch.client.Client; @@ -95,9 +95,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.Priority; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.regex.Regex; @@ -124,7 +124,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.http.HttpInfo; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergePolicyConfig; @@ -142,7 +142,7 @@ import org.opensearch.node.NodeMocksPlugin; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestCancellableNodeClient; import org.opensearch.script.MockScriptService; import org.opensearch.search.MockSearchService; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java index 91c48f1679f9a..1d7c04227b208 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java @@ -56,7 +56,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.indices.IndicesService; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 1bea22852df4c..24067e08a3d89 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -69,16 +69,17 @@ import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Numbers; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.Index; import org.opensearch.common.joda.JodaDeprecationPatterns; import org.opensearch.common.logging.DeprecatedMessage; import org.opensearch.common.logging.HeaderWarning; @@ -107,7 +108,6 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; -import org.opensearch.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; diff --git a/test/framework/src/main/java/org/opensearch/test/RandomObjects.java b/test/framework/src/main/java/org/opensearch/test/RandomObjects.java index 6405aa5ff5c95..e81f0df9a5ac4 100644 --- a/test/framework/src/main/java/org/opensearch/test/RandomObjects.java +++ b/test/framework/src/main/java/org/opensearch/test/RandomObjects.java @@ -44,8 +44,8 @@ import org.opensearch.action.support.replication.ReplicationResponse.ShardInfo.Failure; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.coordination.NoClusterManagerBlockService; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContent; @@ -54,9 +54,9 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.shard.IndexShardRecoveringException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardNotFoundException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java b/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java index 644274b135b7e..54cabf753012b 100644 --- a/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java @@ -32,7 +32,7 @@ package org.opensearch.test; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.io.Streams; diff --git a/test/framework/src/main/java/org/opensearch/test/TestCluster.java b/test/framework/src/main/java/org/opensearch/test/TestCluster.java index 3dcaaefea61f9..61742cd4fb827 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/TestCluster.java @@ -42,7 +42,7 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.index.IndexNotFoundException; import org.opensearch.indices.IndexTemplateMissingException; import org.opensearch.repositories.RepositoryMissingException; diff --git a/test/framework/src/main/java/org/opensearch/test/TestCustomMetadata.java b/test/framework/src/main/java/org/opensearch/test/TestCustomMetadata.java index 127030134b06d..f718de030ea49 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestCustomMetadata.java +++ b/test/framework/src/main/java/org/opensearch/test/TestCustomMetadata.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.AbstractNamedDiffable; import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java index 82e16e6c13005..60425077b14a0 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java @@ -48,7 +48,7 @@ import org.opensearch.index.query.ParsedQuery; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.search.SearchExtBuilder; import org.opensearch.search.SearchShardTarget; diff --git a/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java index 3f8bfe8e9e935..75a86d3d9a0bb 100644 --- a/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java @@ -32,7 +32,7 @@ package org.opensearch.test; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java b/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java index 1ff7a287b9b30..37843ce7ff366 100644 --- a/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java +++ b/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java @@ -40,7 +40,7 @@ import org.opensearch.action.ActionType; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; diff --git a/test/framework/src/main/java/org/opensearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/opensearch/test/engine/MockEngineSupport.java index f2427dfda2742..f232a2b9e4734 100644 --- a/test/framework/src/main/java/org/opensearch/test/engine/MockEngineSupport.java +++ b/test/framework/src/main/java/org/opensearch/test/engine/MockEngineSupport.java @@ -45,7 +45,7 @@ import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineException; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.Closeable; diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java index a36dc26685eb4..b1079ac02f266 100644 --- a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java +++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java @@ -42,7 +42,7 @@ import org.opensearch.gateway.PrimaryShardAllocator; import org.opensearch.gateway.ReplicaShardAllocator; import org.opensearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata.NodeStoreFilesMetadata; diff --git a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java index e52a6b48e219d..230cc50f427f9 100644 --- a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java +++ b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java @@ -49,7 +49,7 @@ import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.action.support.master.AcknowledgedRequestBuilder; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -58,7 +58,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.common.Nullable; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; @@ -66,7 +66,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.suggest.Suggest; import org.opensearch.test.NotEqualMessageBuilder; @@ -602,7 +602,7 @@ public static void assertFutureThrows(ActionFuture futu * Run future.actionGet() and check that it throws an exception of the right type, optionally checking the exception's rest status * * @param exceptionClass expected exception class - * @param status {@link org.opensearch.rest.RestStatus} to check for. Can be null to disable the check + * @param status {@link RestStatus} to check for. Can be null to disable the check * @param extraInfo extra information to add to the failure message. Can be null. */ public static void assertFutureThrows( diff --git a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestChannel.java b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestChannel.java index a39fdd1eae096..6d7951f31a44a 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestChannel.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestChannel.java @@ -34,7 +34,7 @@ import org.opensearch.rest.AbstractRestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java index 293bc4ef1093b..375768ed12cc4 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java @@ -33,15 +33,15 @@ package org.opensearch.test.rest; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; import org.opensearch.http.HttpResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.net.InetSocketAddress; import java.util.Collections; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java index 6f9315ee02a1e..512369fdfe8a1 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java @@ -76,7 +76,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.ReplicationTracker; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.yaml.ObjectPath; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 6cd03a3cf7256..cb460f17cbec0 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -41,7 +41,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.client.NodeSelector; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java index 5969ba8565d51..5c6190ab860b2 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java @@ -36,7 +36,7 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Response; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java index eae4070d56b44..8b62f0d0a9783 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java @@ -33,8 +33,8 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Response; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java index f21fe0cf89ac3..d8a9719fcd9c8 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.restspec; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSection.java index 49ebad356f70e..6902e011f2580 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -31,7 +31,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuite.java index 038d0dd798182..961d931f43c73 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.section; import org.opensearch.client.NodeSelector; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParseException; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java index 9c2a215776a27..29330c47f7d4d 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java @@ -38,7 +38,7 @@ import org.opensearch.client.HasAttributeNodeSelector; import org.opensearch.client.Node; import org.opensearch.client.NodeSelector; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.HeaderWarning; import org.opensearch.core.common.Strings; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java index 2f4c3518b8549..a561a53119a96 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java @@ -31,7 +31,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.rest.yaml.ClientYamlTestExecutionContext; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SkipSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SkipSection.java index 172c509eab0c0..cf0d6dc826ffb 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SkipSection.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SkipSection.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.section; import org.opensearch.Version; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.VersionUtils; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TeardownSection.java index c03ca3d83f35b..88a1f85f7c920 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TeardownSection.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TeardownSection.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TransformAndSetSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TransformAndSetSection.java index 81e62d585378f..f2ecbe4c19806 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TransformAndSetSection.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/TransformAndSetSection.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.rest.yaml.ClientYamlTestExecutionContext; diff --git a/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java b/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java index e38b62c419334..ac4103c8dbbee 100644 --- a/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java +++ b/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java @@ -49,7 +49,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.FsDirectoryFactory; import org.opensearch.index.store.Store; diff --git a/test/framework/src/main/java/org/opensearch/test/store/MockFSIndexStore.java b/test/framework/src/main/java/org/opensearch/test/store/MockFSIndexStore.java index 88e4d0909a6fa..46ca6c456573e 100644 --- a/test/framework/src/main/java/org/opensearch/test/store/MockFSIndexStore.java +++ b/test/framework/src/main/java/org/opensearch/test/store/MockFSIndexStore.java @@ -42,7 +42,7 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.plugins.Plugin; diff --git a/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java index 5b194a828a874..36f3e6cb4b692 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java @@ -38,9 +38,9 @@ import org.opensearch.common.Randomness; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; diff --git a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java index cff363cabf5a3..87f198fb40ee5 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java @@ -41,7 +41,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index fed14d6bbc1ba..29619a541722c 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -50,8 +50,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.network.CloseableChannel; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.network.NetworkUtils; diff --git a/test/framework/src/main/java/org/opensearch/transport/FakeTcpChannel.java b/test/framework/src/main/java/org/opensearch/transport/FakeTcpChannel.java index 73694bc88bd67..f7571c3e92081 100644 --- a/test/framework/src/main/java/org/opensearch/transport/FakeTcpChannel.java +++ b/test/framework/src/main/java/org/opensearch/transport/FakeTcpChannel.java @@ -32,7 +32,7 @@ package org.opensearch.transport; import org.opensearch.action.ActionListener; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.concurrent.CompletableContext; import java.net.InetSocketAddress; diff --git a/test/framework/src/main/java/org/opensearch/transport/TestRequest.java b/test/framework/src/main/java/org/opensearch/transport/TestRequest.java index 29a907a45a925..2fe917235e948 100644 --- a/test/framework/src/main/java/org/opensearch/transport/TestRequest.java +++ b/test/framework/src/main/java/org/opensearch/transport/TestRequest.java @@ -31,8 +31,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/transport/TestResponse.java b/test/framework/src/main/java/org/opensearch/transport/TestResponse.java index ba6db406033dd..09dd50d656004 100644 --- a/test/framework/src/main/java/org/opensearch/transport/TestResponse.java +++ b/test/framework/src/main/java/org/opensearch/transport/TestResponse.java @@ -31,8 +31,8 @@ package org.opensearch.transport; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java index cc6f08d427860..bfacf9d3f2080 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java @@ -41,10 +41,10 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.bytes.CompositeBytesReference; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.recycler.Recycler; import org.opensearch.common.settings.Settings; diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java index 01bd098487c9f..e51f96a86bebf 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java @@ -32,7 +32,7 @@ package org.opensearch.transport.nio; import org.opensearch.Version; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; diff --git a/test/framework/src/test/java/org/opensearch/test/AbstractXContentTestCaseTests.java b/test/framework/src/test/java/org/opensearch/test/AbstractXContentTestCaseTests.java index 494f7c906628f..f47e0f9ea75b5 100644 --- a/test/framework/src/test/java/org/opensearch/test/AbstractXContentTestCaseTests.java +++ b/test/framework/src/test/java/org/opensearch/test/AbstractXContentTestCaseTests.java @@ -34,7 +34,7 @@ import com.carrotsearch.randomizedtesting.RandomizedContext; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; diff --git a/test/framework/src/test/java/org/opensearch/test/XContentTestUtilsTests.java b/test/framework/src/test/java/org/opensearch/test/XContentTestUtilsTests.java index d430d7f2fbcb3..4e1cb8debb4fc 100644 --- a/test/framework/src/test/java/org/opensearch/test/XContentTestUtilsTests.java +++ b/test/framework/src/test/java/org/opensearch/test/XContentTestUtilsTests.java @@ -32,7 +32,7 @@ package org.opensearch.test; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java index e96fb4bab0453..9bddaf013aef4 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.coordination.DeterministicTaskQueue; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.node.Node; diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java index 20ce592bbfd46..a5112bc958954 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/test/framework/src/test/java/org/opensearch/test/hamcrest/OpenSearchAssertionsTests.java b/test/framework/src/test/java/org/opensearch/test/hamcrest/OpenSearchAssertionsTests.java index ba4ded7fcf604..5932f8dfaceec 100644 --- a/test/framework/src/test/java/org/opensearch/test/hamcrest/OpenSearchAssertionsTests.java +++ b/test/framework/src/test/java/org/opensearch/test/hamcrest/OpenSearchAssertionsTests.java @@ -32,12 +32,12 @@ package org.opensearch.test.hamcrest; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.cluster.block.ClusterBlock; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java index 2465e48b9afac..a6482ed09b253 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java @@ -31,7 +31,7 @@ package org.opensearch.test.rest.yaml; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java index 222039e251d99..24d0a1f7f23a7 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java @@ -31,7 +31,7 @@ package org.opensearch.test.rest.yaml.restspec; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.test.OpenSearchTestCase; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index fd9987a683a4f..00984e30df497 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -33,7 +33,7 @@ package org.opensearch.test.rest.yaml.section; import org.opensearch.LegacyESVersion; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.yaml.YamlXContent; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java index 109487b0a75c1..313867abd7c76 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java @@ -33,7 +33,7 @@ package org.opensearch.test.rest.yaml.section; import org.opensearch.client.NodeSelector; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.common.xcontent.yaml.YamlXContent; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/DoSectionTests.java index a2eea80b37b22..4e578ca9fbadf 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/DoSectionTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.client.Node; import org.opensearch.client.NodeSelector; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.logging.HeaderWarning; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetSectionTests.java index c82694f23aa1b..fafde9c64f6e8 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetSectionTests.java @@ -31,7 +31,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.xcontent.yaml.YamlXContent; import static org.hamcrest.Matchers.equalTo; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java index 5cdeeb70c1950..4f0de3e65808c 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java @@ -34,7 +34,7 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.test.VersionUtils; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/TransformAndSetSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/TransformAndSetSectionTests.java index 4eb51d8c6f9bb..b887993503874 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/TransformAndSetSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/TransformAndSetSectionTests.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.test.rest.yaml.ClientYamlTestExecutionContext; import org.opensearch.test.rest.yaml.Stash; diff --git a/test/framework/src/test/java/org/opensearch/test/test/OpenSearchTestCaseTests.java b/test/framework/src/test/java/org/opensearch/test/test/OpenSearchTestCaseTests.java index ff862a5ae673e..65e0a2dfa6d8a 100644 --- a/test/framework/src/test/java/org/opensearch/test/test/OpenSearchTestCaseTests.java +++ b/test/framework/src/test/java/org/opensearch/test/test/OpenSearchTestCaseTests.java @@ -34,7 +34,7 @@ import junit.framework.AssertionFailedError; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.time.DateFormatter; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java b/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java index e42c7e6fc4f85..8b0ffb2d0652d 100644 --- a/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; From 6a35ffa96ec4b5fb67aaeba5d40e9faa8dfb3c26 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 12 Jul 2023 17:58:44 -0500 Subject: [PATCH 116/186] Add 2.10.0 to bwc versions. (#8670) Forward port of #8667. Also, the version `2.9.99` has been used in several places to mean 'the last possible version before 3.0.0' which is not true given that 2.10.0 exists. This commit uses `2.99.99` instead. (Maybe I'll kick myself if we do 90 more minor version releases on the 2.x line?) Signed-off-by: Andrew Ross --- .ci/bwcVersions | 1 + .../src/main/java/org/opensearch/Version.java | 1 + .../test/cluster.put_settings/10_basic.yml | 2 +- .../test/indices.clone/10_basic.yml | 2 +- .../indices.clone/40_wait_for_completion.yml | 2 +- .../20_wait_for_completion.yml | 2 +- .../test/indices.open/30_wait_for_completion.yml | 2 +- .../test/indices.shrink/10_basic.yml | 2 +- .../test/indices.shrink/40_max_shard_size.yml | 2 +- .../indices.shrink/50_wait_for_completion.yml | 2 +- .../test/indices.split/10_basic.yml | 2 +- .../indices.split/40_wait_for_completion.yml | 2 +- .../rest-api-spec/test/pit/10_basic.yml | 2 +- .../180_percentiles_tdigest_metric.yml | 16 ++++++++-------- .../180_percentiles_tdigest_metric_unsigned.yml | 14 +++++++------- 15 files changed, 28 insertions(+), 26 deletions(-) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 7fc76dc8c6c04..31b1cb5efe3a1 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -21,3 +21,4 @@ BWC_VERSION: - "2.8.0" - "2.8.1" - "2.9.0" + - "2.10.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 8c0e247bc3169..f85da63bdbb1f 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -90,6 +90,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version CURRENT = V_3_0_0; diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 4c263ac9f743a..2bc5e98465e16 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -85,7 +85,7 @@ "Test set invalid search backpressure mode": - skip: - version: "- 2.9.99" + version: "- 2.99.99" reason: "Parsing and validation of SearchBackpressureMode does not exist in versions < 3.0" - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml index d92fb434ba718..fa48820a71a89 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml @@ -117,7 +117,7 @@ setup: "Returns error if target index's metadata write is blocked": - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" # block source index's write operations diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/40_wait_for_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/40_wait_for_completion.yml index 88bdb9722c541..b298575d15410 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/40_wait_for_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/40_wait_for_completion.yml @@ -4,7 +4,7 @@ # will return a task immediately and the clone operation will run in background. - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/20_wait_for_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/20_wait_for_completion.yml index 27a2b759dda09..9561ecd89fdad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/20_wait_for_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/20_wait_for_completion.yml @@ -4,7 +4,7 @@ # will return a task immediately and the merge process will run in background. - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/30_wait_for_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/30_wait_for_completion.yml index 93d557ee0aa3f..2caf604eb4296 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/30_wait_for_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/30_wait_for_completion.yml @@ -4,7 +4,7 @@ # will return a task immediately and the open operation will run in background. - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml index 3b89d5c93acd8..426729e737978 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml @@ -76,7 +76,7 @@ setup: "Returns error if target index's metadata write is blocked": - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" # block source index's write operations diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/40_max_shard_size.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/40_max_shard_size.yml index b5fe43edcb003..32ac11097d3dc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/40_max_shard_size.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/40_max_shard_size.yml @@ -4,7 +4,7 @@ # number_of_shards for the target index. - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/50_wait_for_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/50_wait_for_completion.yml index 7ed90cd242223..f7568b1446967 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/50_wait_for_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/50_wait_for_completion.yml @@ -4,7 +4,7 @@ # will return a task immediately and the shrink operation will run in background. - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 5e1cae0d8efc5..50c2819eac9d5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -223,7 +223,7 @@ setup: "Returns error if target index's metadata write is blocked": - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" # block source index's write operations diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/40_wait_for_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/40_wait_for_completion.yml index 9fe1b453501eb..2ce4fc620742a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/40_wait_for_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/40_wait_for_completion.yml @@ -4,7 +4,7 @@ # will return a task immediately and the split operation will run in background. - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "only available in 3.0+" features: allowed_warnings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/pit/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/pit/10_basic.yml index 3637ca7604934..9881869301e2b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/pit/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/pit/10_basic.yml @@ -96,7 +96,7 @@ --- "Delete all": - skip: - version: " - 2.9.99" + version: " - 2.99.99" reason: "mode to be introduced later than 3.0" - do: indices.create: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml index 53d0ed1b2d05f..ff301e6e8ca31 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml @@ -53,7 +53,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -88,7 +88,7 @@ setup: "Basic 3.x test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" @@ -129,7 +129,7 @@ setup: "Compression test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" @@ -174,7 +174,7 @@ setup: "Only aggs test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" @@ -216,7 +216,7 @@ setup: "Filtered test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" @@ -303,7 +303,7 @@ setup: "Metadata test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" @@ -395,7 +395,7 @@ setup: "Explicit Percents test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" @@ -431,7 +431,7 @@ setup: "Non-keyed test": - skip: - version: "- 2.9.99" + version: "- 2.99.99" features: node_selector reason: "t-digest 3.2 was interpolating leading to incorrect percentiles" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric_unsigned.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric_unsigned.yml index 0771b67fddb73..b572be74e908b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric_unsigned.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric_unsigned.yml @@ -56,7 +56,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -136,7 +136,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -224,7 +224,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -306,7 +306,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -398,7 +398,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -464,7 +464,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: @@ -534,7 +534,7 @@ setup: - do: node_selector: - version: "- 2.9.99" + version: "- 2.99.99" search: rest_total_hits_as_int: true body: From 2f4545aafef5f76103574c4a343a68bf54cac7a1 Mon Sep 17 00:00:00 2001 From: Raghuvansh Raj Date: Thu, 13 Jul 2023 09:58:59 +0530 Subject: [PATCH 117/186] [Remote Store] Integrate S3 async upload with S3BlobContainer (#7218) Signed-off-by: Raghuvansh Raj --- .../s3/S3BlobStoreRepositoryTests.java | 2 +- .../repositories/s3/S3BlobContainer.java | 46 +- .../repositories/s3/S3BlobStore.java | 40 +- .../repositories/s3/S3Repository.java | 61 +- .../repositories/s3/S3RepositoryPlugin.java | 118 +++- .../s3/RepositoryCredentialsTests.java | 4 +- .../s3/S3BlobContainerMockClientTests.java | 542 ++++++++++++++++++ .../s3/S3BlobContainerRetriesTests.java | 142 ++++- .../s3/S3ClientSettingsTests.java | 6 +- .../repositories/s3/S3RepositoryTests.java | 7 +- 10 files changed, 948 insertions(+), 20 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index 61268cf00a77a..3070c654a96ee 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -172,7 +172,7 @@ protected S3Repository createRepository( ClusterService clusterService, RecoverySettings recoverySettings ) { - return new S3Repository(metadata, registry, service, clusterService, recoverySettings) { + return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, false) { @Override public BlobStore blobStore() { diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index 49ebce77a59ad..81a902a6992d8 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -39,11 +39,15 @@ import org.opensearch.action.ActionListener; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; +import org.opensearch.common.StreamContext; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.blobstore.DeleteResult; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.support.AbstractBlobContainer; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.collect.Tuple; @@ -72,6 +76,8 @@ import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; import org.opensearch.core.common.Strings; +import org.opensearch.repositories.s3.async.UploadRequest; +import software.amazon.awssdk.services.s3.S3AsyncClient; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -82,6 +88,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; @@ -90,12 +97,13 @@ import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE_USING_MULTIPART; import static org.opensearch.repositories.s3.S3Repository.MIN_PART_SIZE_USING_MULTIPART; -class S3BlobContainer extends AbstractBlobContainer { +class S3BlobContainer extends AbstractBlobContainer implements VerifyingMultiStreamBlobContainer { private static final Logger logger = LogManager.getLogger(S3BlobContainer.class); /** * Maximum number of deletes in a {@link DeleteObjectsRequest}. + * * @see S3 Documentation. */ private static final int MAX_BULK_DELETES = 1000; @@ -166,6 +174,42 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize, b }); } + @Override + public void asyncBlobUpload(WriteContext writeContext, ActionListener completionListener) throws IOException { + UploadRequest uploadRequest = new UploadRequest( + blobStore.bucket(), + buildKey(writeContext.getFileName()), + writeContext.getFileSize(), + writeContext.getWritePriority(), + writeContext.getUploadFinalizer(), + writeContext.doRemoteDataIntegrityCheck(), + writeContext.getExpectedChecksum() + ); + try { + long partSize = blobStore.getAsyncTransferManager().calculateOptimalPartSize(writeContext.getFileSize()); + StreamContext streamContext = SocketAccess.doPrivileged(() -> writeContext.getStreamProvider(partSize)); + try (AmazonAsyncS3Reference amazonS3Reference = SocketAccess.doPrivileged(blobStore::asyncClientReference)) { + + S3AsyncClient s3AsyncClient = writeContext.getWritePriority() == WritePriority.HIGH + ? amazonS3Reference.get().priorityClient() + : amazonS3Reference.get().client(); + CompletableFuture completableFuture = blobStore.getAsyncTransferManager() + .uploadObject(s3AsyncClient, uploadRequest, streamContext); + completableFuture.whenComplete((response, throwable) -> { + if (throwable == null) { + completionListener.onResponse(response); + } else { + Exception ex = throwable instanceof Error ? new Exception(throwable) : (Exception) throwable; + completionListener.onFailure(ex); + } + }); + } + } catch (Exception e) { + logger.info("exception error from blob container for file {}", writeContext.getFileName()); + throw new IOException(e); + } + } + // package private for testing long getLargeBlobThresholdInBytes() { return blobStore.bufferSizeInBytes(); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index 6a9be2df2bf72..30040e182cbc9 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -42,6 +42,8 @@ import org.opensearch.common.unit.ByteSizeValue; import software.amazon.awssdk.services.s3.model.ObjectCannedACL; import software.amazon.awssdk.services.s3.model.StorageClass; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferManager; import java.io.IOException; import java.util.Locale; @@ -53,6 +55,8 @@ class S3BlobStore implements BlobStore { private final S3Service service; + private final S3AsyncService s3AsyncService; + private final String bucket; private final ByteSizeValue bufferSize; @@ -67,22 +71,41 @@ class S3BlobStore implements BlobStore { private final StatsMetricPublisher statsMetricPublisher = new StatsMetricPublisher(); + private final AsyncTransferManager asyncTransferManager; + private final AsyncExecutorContainer priorityExecutorBuilder; + private final AsyncExecutorContainer normalExecutorBuilder; + private final boolean multipartUploadEnabled; + S3BlobStore( S3Service service, + S3AsyncService s3AsyncService, + boolean multipartUploadEnabled, String bucket, boolean serverSideEncryption, ByteSizeValue bufferSize, String cannedACL, String storageClass, - RepositoryMetadata repositoryMetadata + RepositoryMetadata repositoryMetadata, + AsyncTransferManager asyncTransferManager, + AsyncExecutorContainer priorityExecutorBuilder, + AsyncExecutorContainer normalExecutorBuilder ) { this.service = service; + this.s3AsyncService = s3AsyncService; + this.multipartUploadEnabled = multipartUploadEnabled; this.bucket = bucket; this.serverSideEncryption = serverSideEncryption; this.bufferSize = bufferSize; this.cannedACL = initCannedACL(cannedACL); this.storageClass = initStorageClass(storageClass); this.repositoryMetadata = repositoryMetadata; + this.asyncTransferManager = asyncTransferManager; + this.normalExecutorBuilder = normalExecutorBuilder; + this.priorityExecutorBuilder = priorityExecutorBuilder; + } + + public boolean isMultipartUploadEnabled() { + return multipartUploadEnabled; } @Override @@ -94,6 +117,10 @@ public AmazonS3Reference clientReference() { return service.client(repositoryMetadata); } + public AmazonAsyncS3Reference asyncClientReference() { + return s3AsyncService.client(repositoryMetadata, priorityExecutorBuilder, normalExecutorBuilder); + } + int getMaxRetries() { return service.settings(repositoryMetadata).maxRetries; } @@ -117,7 +144,12 @@ public BlobContainer blobContainer(BlobPath path) { @Override public void close() throws IOException { - this.service.close(); + if (service != null) { + this.service.close(); + } + if (s3AsyncService != null) { + this.s3AsyncService.close(); + } } @Override @@ -170,4 +202,8 @@ public static ObjectCannedACL initCannedACL(String cannedACL) { throw new BlobStoreException("cannedACL is not valid: [" + cannedACL + "]"); } + + public AsyncTransferManager getAsyncTransferManager() { + return asyncTransferManager; + } } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index 07abb69c11bdd..d42bfc0be7e4f 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -34,7 +34,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.ClusterState; @@ -57,6 +56,8 @@ import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.ShardGenerations; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferManager; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.threadpool.Scheduler; @@ -103,6 +104,11 @@ class S3Repository extends MeteredBlobStoreRepository { ByteSizeUnit.BYTES ); + private static final ByteSizeValue DEFAULT_MULTIPART_UPLOAD_MINIMUM_PART_SIZE = new ByteSizeValue( + ByteSizeUnit.MB.toBytes(16), + ByteSizeUnit.BYTES + ); + static final Setting BUCKET_SETTING = Setting.simpleString("bucket"); /** @@ -146,6 +152,26 @@ class S3Repository extends MeteredBlobStoreRepository { MAX_PART_SIZE_USING_MULTIPART ); + /** + * Minimum part size for parallel multipart uploads + */ + static final Setting PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING = Setting.byteSizeSetting( + "parallel_multipart_upload.minimum_part_size", + DEFAULT_MULTIPART_UPLOAD_MINIMUM_PART_SIZE, + MIN_PART_SIZE_USING_MULTIPART, + MAX_PART_SIZE_USING_MULTIPART, + Setting.Property.NodeScope + ); + + /** + * This setting controls whether parallel multipart uploads will be used when calling S3 or not + */ + public static Setting PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING = Setting.boolSetting( + "parallel_multipart_upload.enabled", + true, + Setting.Property.NodeScope + ); + /** * Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g. */ @@ -193,6 +219,12 @@ class S3Repository extends MeteredBlobStoreRepository { private final RepositoryMetadata repositoryMetadata; + private final AsyncTransferManager asyncUploadUtils; + private final S3AsyncService s3AsyncService; + private final boolean multipartUploadEnabled; + private final AsyncExecutorContainer priorityExecutorBuilder; + private final AsyncExecutorContainer normalExecutorBuilder; + /** * Constructs an s3 backed repository */ @@ -201,7 +233,12 @@ class S3Repository extends MeteredBlobStoreRepository { final NamedXContentRegistry namedXContentRegistry, final S3Service service, final ClusterService clusterService, - final RecoverySettings recoverySettings + final RecoverySettings recoverySettings, + final AsyncTransferManager asyncUploadUtils, + final AsyncExecutorContainer priorityExecutorBuilder, + final AsyncExecutorContainer normalExecutorBuilder, + final S3AsyncService s3AsyncService, + final boolean multipartUploadEnabled ) { super( metadata, @@ -212,8 +249,13 @@ class S3Repository extends MeteredBlobStoreRepository { buildLocation(metadata) ); this.service = service; + this.s3AsyncService = s3AsyncService; + this.multipartUploadEnabled = multipartUploadEnabled; this.repositoryMetadata = metadata; + this.asyncUploadUtils = asyncUploadUtils; + this.priorityExecutorBuilder = priorityExecutorBuilder; + this.normalExecutorBuilder = normalExecutorBuilder; // Parse and validate the user's S3 Storage Class setting this.bucket = BUCKET_SETTING.get(metadata.settings()); @@ -314,7 +356,20 @@ public void deleteSnapshots( @Override protected S3BlobStore createBlobStore() { - return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, repositoryMetadata); + return new S3BlobStore( + service, + s3AsyncService, + multipartUploadEnabled, + bucket, + serverSideEncryption, + bufferSize, + cannedACL, + storageClass, + repositoryMetadata, + asyncUploadUtils, + priorityExecutorBuilder, + normalExecutorBuilder + ); } // only use for testing diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 828bf85fd7889..30f792346f9be 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -32,44 +32,131 @@ package org.opensearch.repositories.s3; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ReloadablePlugin; import org.opensearch.plugins.RepositoryPlugin; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup; +import org.opensearch.repositories.s3.async.AsyncTransferManager; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.FixedExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; /** * A plugin to add a repository type that writes to and from the AWS S3. */ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin { + private static final String PRIORITY_FUTURE_COMPLETION = "priority_future_completion"; + private static final String PRIORITY_STREAM_READER = "priority_stream_reader"; + private static final String FUTURE_COMPLETION = "future_completion"; + private static final String STREAM_READER = "stream_reader"; protected final S3Service service; + private final S3AsyncService s3AsyncService; + private final Path configPath; + private AsyncExecutorContainer priorityExecutorBuilder; + private AsyncExecutorContainer normalExecutorBuilder; + public S3RepositoryPlugin(final Settings settings, final Path configPath) { - this(settings, configPath, new S3Service(configPath)); + this(settings, configPath, new S3Service(configPath), new S3AsyncService(configPath)); + } + + @Override + public List> getExecutorBuilders(Settings settings) { + List> executorBuilders = new ArrayList<>(); + executorBuilders.add( + new FixedExecutorBuilder(settings, PRIORITY_FUTURE_COMPLETION, priorityPoolCount(settings), 10_000, PRIORITY_FUTURE_COMPLETION) + ); + executorBuilders.add( + new FixedExecutorBuilder(settings, PRIORITY_STREAM_READER, priorityPoolCount(settings), 10_000, PRIORITY_STREAM_READER) + ); + executorBuilders.add(new FixedExecutorBuilder(settings, FUTURE_COMPLETION, normalPoolCount(settings), 10_000, FUTURE_COMPLETION)); + executorBuilders.add(new FixedExecutorBuilder(settings, STREAM_READER, normalPoolCount(settings), 10_000, STREAM_READER)); + return executorBuilders; } - S3RepositoryPlugin(final Settings settings, final Path configPath, final S3Service service) { + S3RepositoryPlugin(final Settings settings, final Path configPath, final S3Service service, final S3AsyncService s3AsyncService) { this.service = Objects.requireNonNull(service, "S3 service must not be null"); this.configPath = configPath; // eagerly load client settings so that secure settings are read - final Map clientsSettings = S3ClientSettings.load(settings, configPath); + Map clientsSettings = S3ClientSettings.load(settings, configPath); + this.s3AsyncService = Objects.requireNonNull(s3AsyncService, "S3AsyncService must not be null"); this.service.refreshAndClearCache(clientsSettings); + this.s3AsyncService.refreshAndClearCache(clientsSettings); + } + + private static int boundedBy(int value, int min, int max) { + return Math.min(max, Math.max(min, value)); + } + + private static int allocatedProcessors(Settings settings) { + return OpenSearchExecutors.allocatedProcessors(settings); + } + + private static int priorityPoolCount(Settings settings) { + return boundedBy((allocatedProcessors(settings) + 1) / 2, 2, 4); + } + + private static int normalPoolCount(Settings settings) { + return boundedBy((allocatedProcessors(settings) + 7) / 8, 1, 2); + } + + @Override + public Collection createComponents( + final Client client, + final ClusterService clusterService, + final ThreadPool threadPool, + final ResourceWatcherService resourceWatcherService, + final ScriptService scriptService, + final NamedXContentRegistry xContentRegistry, + final Environment environment, + final NodeEnvironment nodeEnvironment, + final NamedWriteableRegistry namedWriteableRegistry, + final IndexNameExpressionResolver expressionResolver, + final Supplier repositoriesServiceSupplier + ) { + int priorityEventLoopThreads = priorityPoolCount(clusterService.getSettings()); + int normalEventLoopThreads = normalPoolCount(clusterService.getSettings()); + this.priorityExecutorBuilder = new AsyncExecutorContainer( + threadPool.executor(PRIORITY_FUTURE_COMPLETION), + threadPool.executor(PRIORITY_STREAM_READER), + new AsyncTransferEventLoopGroup(priorityEventLoopThreads) + ); + this.normalExecutorBuilder = new AsyncExecutorContainer( + threadPool.executor(FUTURE_COMPLETION), + threadPool.executor(STREAM_READER), + new AsyncTransferEventLoopGroup(normalEventLoopThreads) + ); + return Collections.emptyList(); } // proxy method for testing @@ -79,7 +166,24 @@ protected S3Repository createRepository( final ClusterService clusterService, final RecoverySettings recoverySettings ) { - return new S3Repository(metadata, registry, service, clusterService, recoverySettings); + + AsyncTransferManager asyncUploadUtils = new AsyncTransferManager( + S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.get(clusterService.getSettings()).getBytes(), + normalExecutorBuilder.getStreamReader(), + priorityExecutorBuilder.getStreamReader() + ); + return new S3Repository( + metadata, + registry, + service, + clusterService, + recoverySettings, + asyncUploadUtils, + priorityExecutorBuilder, + normalExecutorBuilder, + s3AsyncService, + S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING.get(clusterService.getSettings()) + ); } @Override @@ -119,7 +223,9 @@ public List> getSettings() { S3ClientSettings.REGION, S3ClientSettings.ROLE_ARN_SETTING, S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING, - S3ClientSettings.ROLE_SESSION_NAME_SETTING + S3ClientSettings.ROLE_SESSION_NAME_SETTING, + S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING, + S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING ); } @@ -128,10 +234,12 @@ public void reload(Settings settings) { // secure settings should be readable final Map clientsSettings = S3ClientSettings.load(settings, configPath); service.refreshAndClearCache(clientsSettings); + s3AsyncService.refreshAndClearCache(clientsSettings); } @Override public void close() throws IOException { service.close(); + s3AsyncService.close(); } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java index 3ccf6553c479d..46e589f7fa41f 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java @@ -291,7 +291,7 @@ private void createRepository(final String name, final Settings repositorySettin public static final class ProxyS3RepositoryPlugin extends S3RepositoryPlugin { public ProxyS3RepositoryPlugin(Settings settings, Path configPath) { - super(settings, configPath, new ProxyS3Service(configPath)); + super(settings, configPath, new ProxyS3Service(configPath), new S3AsyncService(configPath)); } @Override @@ -301,7 +301,7 @@ protected S3Repository createRepository( ClusterService clusterService, RecoverySettings recoverySettings ) { - return new S3Repository(metadata, registry, service, clusterService, recoverySettings) { + return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, false) { @Override protected void assertSnapshotOrGenericThread() { // eliminate thread name check as we create repo manually on test/main threads diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java new file mode 100644 index 0000000000000..10137f0475177 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java @@ -0,0 +1,542 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.apache.lucene.store.IndexInput; +import org.junit.After; +import org.junit.Before; +import org.mockito.invocation.InvocationOnMock; +import org.opensearch.action.ActionListener; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.CheckedTriFunction; +import org.opensearch.common.StreamContext; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.StreamContextSupplier; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferManager; +import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup; +import org.opensearch.test.OpenSearchTestCase; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class S3BlobContainerMockClientTests extends OpenSearchTestCase implements ConfigPathSupport { + + private MockS3AsyncService asyncService; + private ExecutorService futureCompletionService; + private ExecutorService streamReaderService; + private AsyncTransferEventLoopGroup transferNIOGroup; + private S3BlobContainer blobContainer; + + static class MockS3AsyncService extends S3AsyncService { + + private final S3AsyncClient asyncClient = mock(S3AsyncClient.class); + private final int maxDelayInFutureCompletionMillis; + + private boolean failPutObjectRequest; + private boolean failCreateMultipartUploadRequest; + private boolean failUploadPartRequest; + private boolean failCompleteMultipartUploadRequest; + + private String multipartUploadId; + + public MockS3AsyncService(Path configPath, int maxDelayInFutureCompletionMillis) { + super(configPath); + this.maxDelayInFutureCompletionMillis = maxDelayInFutureCompletionMillis; + } + + public void initializeMocks( + boolean failPutObjectRequest, + boolean failCreateMultipartUploadRequest, + boolean failUploadPartRequest, + boolean failCompleteMultipartUploadRequest + ) { + setupFailureBooleans( + failPutObjectRequest, + failCreateMultipartUploadRequest, + failUploadPartRequest, + failCompleteMultipartUploadRequest + ); + doAnswer(this::doOnPutObject).when(asyncClient).putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)); + doAnswer(this::doOnDeleteObject).when(asyncClient).deleteObject(any(DeleteObjectRequest.class)); + doAnswer(this::doOnCreateMultipartUpload).when(asyncClient).createMultipartUpload(any(CreateMultipartUploadRequest.class)); + doAnswer(this::doOnPartUpload).when(asyncClient).uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class)); + doAnswer(this::doOnCompleteMultipartUpload).when(asyncClient) + .completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + doAnswer(this::doOnAbortMultipartUpload).when(asyncClient).abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + } + + private void setupFailureBooleans( + boolean failPutObjectRequest, + boolean failCreateMultipartUploadRequest, + boolean failUploadPartRequest, + boolean failCompleteMultipartUploadRequest + ) { + this.failPutObjectRequest = failPutObjectRequest; + this.failCreateMultipartUploadRequest = failCreateMultipartUploadRequest; + this.failUploadPartRequest = failUploadPartRequest; + this.failCompleteMultipartUploadRequest = failCompleteMultipartUploadRequest; + } + + private CompletableFuture doOnPutObject(InvocationOnMock invocationOnMock) { + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(randomInt(maxDelayInFutureCompletionMillis)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (failPutObjectRequest) { + completableFuture.completeExceptionally(new IOException()); + } else { + completableFuture.complete(PutObjectResponse.builder().build()); + } + }).start(); + + return completableFuture; + } + + private CompletableFuture doOnDeleteObject(InvocationOnMock invocationOnMock) { + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(randomInt(maxDelayInFutureCompletionMillis)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (failPutObjectRequest) { + completableFuture.completeExceptionally(new IOException()); + } else { + completableFuture.complete(DeleteObjectResponse.builder().build()); + } + }).start(); + + return completableFuture; + } + + private CompletableFuture doOnCreateMultipartUpload(InvocationOnMock invocationOnMock) { + multipartUploadId = randomAlphaOfLength(5); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(randomInt(maxDelayInFutureCompletionMillis)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (failCreateMultipartUploadRequest) { + completableFuture.completeExceptionally(new IOException()); + } else { + completableFuture.complete(CreateMultipartUploadResponse.builder().uploadId(multipartUploadId).build()); + } + }).start(); + + return completableFuture; + } + + private CompletableFuture doOnPartUpload(InvocationOnMock invocationOnMock) { + UploadPartRequest uploadPartRequest = invocationOnMock.getArgument(0); + assertEquals(multipartUploadId, uploadPartRequest.uploadId()); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(randomInt(maxDelayInFutureCompletionMillis)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (failUploadPartRequest) { + completableFuture.completeExceptionally(new IOException()); + } else { + completableFuture.complete(UploadPartResponse.builder().eTag("eTag").build()); + } + }).start(); + + return completableFuture; + } + + private CompletableFuture doOnCompleteMultipartUpload(InvocationOnMock invocationOnMock) { + CompleteMultipartUploadRequest completeMultipartUploadRequest = invocationOnMock.getArgument(0); + assertEquals(multipartUploadId, completeMultipartUploadRequest.uploadId()); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(randomInt(maxDelayInFutureCompletionMillis)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (failCompleteMultipartUploadRequest) { + completableFuture.completeExceptionally(new IOException()); + } else { + completableFuture.complete(CompleteMultipartUploadResponse.builder().build()); + } + }).start(); + + return completableFuture; + } + + private CompletableFuture doOnAbortMultipartUpload(InvocationOnMock invocationOnMock) { + AbortMultipartUploadRequest abortMultipartUploadRequest = invocationOnMock.getArgument(0); + assertEquals(multipartUploadId, abortMultipartUploadRequest.uploadId()); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(randomInt(maxDelayInFutureCompletionMillis)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + completableFuture.complete(AbortMultipartUploadResponse.builder().build()); + + }).start(); + + return completableFuture; + } + + public void verifyMultipartUploadCallCount(int numberOfParts, boolean finalizeUploadFailure) { + verify(asyncClient, times(1)).createMultipartUpload(any(CreateMultipartUploadRequest.class)); + verify(asyncClient, times(!failCreateMultipartUploadRequest ? numberOfParts : 0)).uploadPart( + any(UploadPartRequest.class), + any(AsyncRequestBody.class) + ); + verify(asyncClient, times(!failCreateMultipartUploadRequest && !failUploadPartRequest && !finalizeUploadFailure ? 1 : 0)) + .completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + verify( + asyncClient, + times( + (!failCreateMultipartUploadRequest && (failUploadPartRequest || failCompleteMultipartUploadRequest)) + || finalizeUploadFailure ? 1 : 0 + ) + ).abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + } + + public void verifySingleChunkUploadCallCount(boolean finalizeUploadFailure) { + verify(asyncClient, times(1)).putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)); + verify(asyncClient, times(finalizeUploadFailure ? 1 : 0)).deleteObject(any(DeleteObjectRequest.class)); + } + + @Override + public AmazonAsyncS3Reference client( + RepositoryMetadata repositoryMetadata, + AsyncExecutorContainer priorityExecutorBuilder, + AsyncExecutorContainer normalExecutorBuilder + ) { + return new AmazonAsyncS3Reference(AmazonAsyncS3WithCredentials.create(asyncClient, asyncClient, null)); + } + } + + /** + * An IndexInput implementation that serves only zeroes + */ + static class ZeroIndexInput extends IndexInput { + + private final AtomicBoolean closed = new AtomicBoolean(false); + private final AtomicLong reads = new AtomicLong(0); + private final long length; + + /** + * @param resourceDescription resourceDescription should be a non-null, opaque string describing this resource; it's returned + * from {@link #toString}. + */ + public ZeroIndexInput(String resourceDescription, final long length) { + super(resourceDescription); + this.length = length; + } + + @Override + public void close() throws IOException { + closed.set(true); + } + + @Override + public long getFilePointer() { + return reads.get(); + } + + @Override + public void seek(long pos) throws IOException { + reads.set(pos); + } + + @Override + public long length() { + return length; + } + + @Override + public IndexInput slice(String sliceDescription, long offset, long length) throws IOException { + return new ZeroIndexInput(sliceDescription, length); + } + + @Override + public byte readByte() throws IOException { + ensureOpen(); + return (byte) ((reads.incrementAndGet() <= length) ? 0 : -1); + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + ensureOpen(); + final long available = available(); + final int toCopy = Math.min(len, (int) available); + Arrays.fill(b, offset, offset + toCopy, (byte) 0); + reads.addAndGet(toCopy); + } + + private long available() { + return Math.max(length - reads.get(), 0); + } + + private void ensureOpen() throws IOException { + if (closed.get()) { + throw new IOException("Stream closed"); + } + } + } + + @Override + @Before + public void setUp() throws Exception { + asyncService = new MockS3AsyncService(configPath(), 1000); + futureCompletionService = Executors.newSingleThreadExecutor(); + streamReaderService = Executors.newSingleThreadExecutor(); + transferNIOGroup = new AsyncTransferEventLoopGroup(1); + blobContainer = createBlobContainer(); + super.setUp(); + } + + @Override + @After + public void tearDown() throws Exception { + IOUtils.close(asyncService); + super.tearDown(); + } + + private S3BlobContainer createBlobContainer() { + return new S3BlobContainer(BlobPath.cleanPath(), createBlobStore()); + } + + private S3BlobStore createBlobStore() { + final String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + + final RepositoryMetadata repositoryMetadata = new RepositoryMetadata( + "repository", + S3Repository.TYPE, + Settings.builder().put(S3Repository.CLIENT_NAME.getKey(), clientName).build() + ); + + AsyncExecutorContainer asyncExecutorContainer = new AsyncExecutorContainer( + futureCompletionService, + streamReaderService, + transferNIOGroup + ); + + return new S3BlobStore( + null, + asyncService, + true, + "bucket", + S3Repository.SERVER_SIDE_ENCRYPTION_SETTING.getDefault(Settings.EMPTY), + S3Repository.BUFFER_SIZE_SETTING.getDefault(Settings.EMPTY), + S3Repository.CANNED_ACL_SETTING.getDefault(Settings.EMPTY), + S3Repository.STORAGE_CLASS_SETTING.getDefault(Settings.EMPTY), + repositoryMetadata, + new AsyncTransferManager( + S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(), + asyncExecutorContainer.getStreamReader(), + asyncExecutorContainer.getStreamReader() + ), + asyncExecutorContainer, + asyncExecutorContainer + ); + } + + public void testWriteBlobByStreamsNoFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(false, false, false, false); + testWriteBlobByStreamsLargeBlob(false, false); + } + + public void testWriteBlobByStreamsFinalizeUploadFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(false, false, false, false); + testWriteBlobByStreamsLargeBlob(false, true); + } + + public void testWriteBlobByStreamsCreateMultipartRequestFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(false, true, false, false); + testWriteBlobByStreamsLargeBlob(true, false); + } + + public void testWriteBlobByStreamsUploadPartRequestFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(false, false, true, false); + testWriteBlobByStreamsLargeBlob(true, false); + } + + public void testWriteBlobByStreamsCompleteMultipartRequestFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(false, false, false, true); + testWriteBlobByStreamsLargeBlob(true, false); + } + + public void testWriteBlobByStreamsSingleChunkUploadNoFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(false, false, false, false); + testWriteBlobByStreams(false, false); + } + + public void testWriteBlobByStreamsSingleChunkUploadPutObjectFailure() throws IOException, ExecutionException, InterruptedException { + asyncService.initializeMocks(true, false, false, false); + testWriteBlobByStreams(true, false); + } + + public void testWriteBlobByStreamsSingleChunkUploadFinalizeUploadFailure() throws IOException, ExecutionException, + InterruptedException { + asyncService.initializeMocks(false, false, false, false); + testWriteBlobByStreams(false, true); + } + + private void testWriteBlobByStreams(boolean expectException, boolean throwExceptionOnFinalizeUpload) throws IOException, + ExecutionException, InterruptedException { + final byte[] bytes = randomByteArrayOfLength(100); + List openInputStreams = new ArrayList<>(); + CountDownLatch countDownLatch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + ActionListener completionListener = ActionListener.wrap(resp -> { countDownLatch.countDown(); }, ex -> { + exceptionRef.set(ex); + countDownLatch.countDown(); + }); + blobContainer.asyncBlobUpload(new WriteContext("write_blob_by_streams_max_retries", new StreamContextSupplier() { + @Override + public StreamContext supplyStreamContext(long partSize) { + return new StreamContext(new CheckedTriFunction() { + @Override + public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException { + InputStream inputStream = new OffsetRangeIndexInputStream(new ByteArrayIndexInput("desc", bytes), size, position); + openInputStreams.add(inputStream); + return new InputStreamContainer(inputStream, size, position); + } + }, partSize, calculateLastPartSize(bytes.length, partSize), calculateNumberOfParts(bytes.length, partSize)); + } + }, bytes.length, false, WritePriority.NORMAL, uploadSuccess -> { + assertTrue(uploadSuccess); + if (throwExceptionOnFinalizeUpload) { + throw new RuntimeException(); + } + }, false, null), completionListener); + + assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS)); + // wait for completableFuture to finish + if (expectException || throwExceptionOnFinalizeUpload) { + assertNotNull(exceptionRef.get()); + } + + asyncService.verifySingleChunkUploadCallCount(throwExceptionOnFinalizeUpload); + + openInputStreams.forEach(inputStream -> { + try { + inputStream.close(); + } catch (IOException e) { + fail("Failure while closing open input streams"); + } + }); + } + + private void testWriteBlobByStreamsLargeBlob(boolean expectException, boolean throwExceptionOnFinalizeUpload) throws IOException, + ExecutionException, InterruptedException { + final ByteSizeValue partSize = S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY); + + int numberOfParts = randomIntBetween(2, 5); + final long lastPartSize = randomLongBetween(10, 512); + final long blobSize = ((numberOfParts - 1) * partSize.getBytes()) + lastPartSize; + CountDownLatch countDownLatch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + ActionListener completionListener = ActionListener.wrap(resp -> { countDownLatch.countDown(); }, ex -> { + exceptionRef.set(ex); + countDownLatch.countDown(); + }); + List openInputStreams = new ArrayList<>(); + blobContainer.asyncBlobUpload(new WriteContext("write_large_blob", new StreamContextSupplier() { + @Override + public StreamContext supplyStreamContext(long partSize) { + return new StreamContext(new CheckedTriFunction() { + @Override + public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException { + InputStream inputStream = new OffsetRangeIndexInputStream(new ZeroIndexInput("desc", blobSize), size, position); + openInputStreams.add(inputStream); + return new InputStreamContainer(inputStream, size, position); + } + }, partSize, calculateLastPartSize(blobSize, partSize), calculateNumberOfParts(blobSize, partSize)); + } + }, blobSize, false, WritePriority.HIGH, uploadSuccess -> { + assertTrue(uploadSuccess); + if (throwExceptionOnFinalizeUpload) { + throw new RuntimeException(); + } + }, false, null), completionListener); + + assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS)); + if (expectException || throwExceptionOnFinalizeUpload) { + assertNotNull(exceptionRef.get()); + } + + asyncService.verifyMultipartUploadCallCount(numberOfParts, throwExceptionOnFinalizeUpload); + + openInputStreams.forEach(inputStream -> { + try { + inputStream.close(); + } catch (IOException ex) { + logger.error("Error closing input stream"); + } + }); + } + + private long calculateLastPartSize(long totalSize, long partSize) { + return totalSize % partSize == 0 ? partSize : totalSize % partSize; + } + + private int calculateNumberOfParts(long contentLength, long partSize) { + return (int) ((contentLength % partSize) == 0 ? contentLength / partSize : (contentLength / partSize) + 1); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 045ce73daf5a3..1a1fb123aa5ea 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -33,14 +33,24 @@ import org.apache.http.HttpStatus; import org.junit.After; +import org.junit.Assert; import org.junit.Before; +import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.CheckedTriFunction; import org.opensearch.common.Nullable; +import org.opensearch.common.StreamContext; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.VerifyingMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.write.StreamContextSupplier; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.hash.MessageDigests; +import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.lucene.store.InputStreamIndexInput; @@ -53,10 +63,13 @@ import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.AbstractBlobContainerRetriesTestCase; +import org.opensearch.repositories.blobstore.ZeroInputStream; +import org.opensearch.repositories.s3.async.AsyncExecutorContainer; +import org.opensearch.repositories.s3.async.AsyncTransferManager; +import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.io.SdkDigestInputStream; import software.amazon.awssdk.utils.internal.Base16; -import org.opensearch.repositories.blobstore.ZeroInputStream; import java.io.ByteArrayInputStream; import java.io.FilterInputStream; @@ -65,9 +78,16 @@ import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -88,17 +108,34 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes private S3Service service; private String previousOpenSearchPathConf; + private S3AsyncService asyncService; + private ExecutorService futureCompletionService; + private ExecutorService streamReaderService; + private AsyncTransferEventLoopGroup transferNIOGroup; @Before public void setUp() throws Exception { previousOpenSearchPathConf = SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", configPath().toString())); service = new S3Service(configPath()); + asyncService = new S3AsyncService(configPath()); + + futureCompletionService = Executors.newSingleThreadExecutor(); + streamReaderService = Executors.newSingleThreadExecutor(); + transferNIOGroup = new AsyncTransferEventLoopGroup(1); + + // needed by S3AsyncService + SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", configPath().toString())); super.setUp(); } @After public void tearDown() throws Exception { - IOUtils.close(service); + IOUtils.close(service, asyncService); + + streamReaderService.shutdown(); + futureCompletionService.shutdown(); + IOUtils.close(transferNIOGroup); + if (previousOpenSearchPathConf != null) { SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", previousOpenSearchPathConf)); } else { @@ -123,7 +160,7 @@ protected Class unresponsiveExceptionType() { } @Override - protected BlobContainer createBlobContainer( + protected VerifyingMultiStreamBlobContainer createBlobContainer( final @Nullable Integer maxRetries, final @Nullable TimeValue readTimeout, final @Nullable Boolean disableChunkedEncoding, @@ -152,6 +189,7 @@ protected BlobContainer createBlobContainer( secureSettings.setString(S3ClientSettings.SECRET_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "secret"); clientSettings.setSecureSettings(secureSettings); service.refreshAndClearCache(S3ClientSettings.load(clientSettings.build(), configPath())); + asyncService.refreshAndClearCache(S3ClientSettings.load(clientSettings.build(), configPath())); final RepositoryMetadata repositoryMetadata = new RepositoryMetadata( "repository", @@ -159,16 +197,31 @@ protected BlobContainer createBlobContainer( Settings.builder().put(S3Repository.CLIENT_NAME.getKey(), clientName).build() ); + AsyncExecutorContainer asyncExecutorContainer = new AsyncExecutorContainer( + futureCompletionService, + streamReaderService, + transferNIOGroup + ); + return new S3BlobContainer( BlobPath.cleanPath(), new S3BlobStore( service, + asyncService, + true, "bucket", S3Repository.SERVER_SIDE_ENCRYPTION_SETTING.getDefault(Settings.EMPTY), bufferSize == null ? S3Repository.BUFFER_SIZE_SETTING.getDefault(Settings.EMPTY) : bufferSize, S3Repository.CANNED_ACL_SETTING.getDefault(Settings.EMPTY), S3Repository.STORAGE_CLASS_SETTING.getDefault(Settings.EMPTY), - repositoryMetadata + repositoryMetadata, + new AsyncTransferManager( + S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(), + asyncExecutorContainer.getStreamReader(), + asyncExecutorContainer.getStreamReader() + ), + asyncExecutorContainer, + asyncExecutorContainer ) ) { @Override @@ -228,6 +281,87 @@ public void testWriteBlobWithRetries() throws Exception { assertThat(countDown.isCountedDown(), is(true)); } + public void testWriteBlobByStreamsWithRetries() throws Exception { + final int maxRetries = randomInt(5); + final CountDown countDown = new CountDown(maxRetries + 1); + + final byte[] bytes = randomBlobContent(); + httpServer.createContext("/bucket/write_blob_by_streams_max_retries", exchange -> { + if ("PUT".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getQuery() == null) { + if (countDown.countDown()) { + final BytesReference body = Streams.readFully(exchange.getRequestBody()); + if (Objects.deepEquals(bytes, BytesReference.toBytes(body))) { + exchange.sendResponseHeaders(HttpStatus.SC_OK, -1); + } else { + exchange.sendResponseHeaders(HttpStatus.SC_BAD_REQUEST, -1); + } + exchange.close(); + return; + } + + if (randomBoolean()) { + if (randomBoolean()) { + Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]); + } else { + Streams.readFully(exchange.getRequestBody()); + exchange.sendResponseHeaders( + randomFrom( + HttpStatus.SC_INTERNAL_SERVER_ERROR, + HttpStatus.SC_BAD_GATEWAY, + HttpStatus.SC_SERVICE_UNAVAILABLE, + HttpStatus.SC_GATEWAY_TIMEOUT + ), + -1 + ); + } + } + exchange.close(); + } + }); + + final VerifyingMultiStreamBlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null); + List openInputStreams = new ArrayList<>(); + CountDownLatch countDownLatch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + ActionListener completionListener = ActionListener.wrap(resp -> { countDownLatch.countDown(); }, ex -> { + exceptionRef.set(ex); + countDownLatch.countDown(); + }); + blobContainer.asyncBlobUpload(new WriteContext("write_blob_by_streams_max_retries", new StreamContextSupplier() { + @Override + public StreamContext supplyStreamContext(long partSize) { + return new StreamContext(new CheckedTriFunction() { + @Override + public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException { + InputStream inputStream = new OffsetRangeIndexInputStream(new ByteArrayIndexInput("desc", bytes), size, position); + openInputStreams.add(inputStream); + return new InputStreamContainer(inputStream, size, position); + } + }, partSize, calculateLastPartSize(bytes.length, partSize), calculateNumberOfParts(bytes.length, partSize)); + } + }, bytes.length, false, WritePriority.NORMAL, Assert::assertTrue, false, null), completionListener); + + assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS)); + + assertThat(countDown.isCountedDown(), is(true)); + + openInputStreams.forEach(inputStream -> { + try { + inputStream.close(); + } catch (IOException e) { + fail("Failure while closing open input streams"); + } + }); + } + + private long calculateLastPartSize(long totalSize, long partSize) { + return totalSize % partSize == 0 ? partSize : totalSize % partSize; + } + + private int calculateNumberOfParts(long contentLength, long partSize) { + return (int) ((contentLength % partSize) == 0 ? contentLength / partSize : (contentLength / partSize) + 1); + } + public void testWriteBlobWithReadTimeouts() { final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128)); final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500)); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java index 130b8efca0512..1edf8d53c1e73 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java @@ -68,7 +68,11 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.endpoint, is(emptyString())); assertThat(defaultSettings.protocol, is(Protocol.HTTPS)); assertThat(defaultSettings.proxySettings, is(ProxySettings.NO_PROXY_SETTINGS)); - assertThat(defaultSettings.readTimeoutMillis, is(50_000)); + assertThat(defaultSettings.readTimeoutMillis, is(50 * 1000)); + assertThat(defaultSettings.requestTimeoutMillis, is(120 * 1000)); + assertThat(defaultSettings.connectionTimeoutMillis, is(10 * 1000)); + assertThat(defaultSettings.connectionTTLMillis, is(5 * 1000)); + assertThat(defaultSettings.maxConnections, is(100)); assertThat(defaultSettings.maxRetries, is(3)); assertThat(defaultSettings.throttleRetries, is(true)); } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index dc63ed50d5f3d..84d56c7ae2854 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -138,7 +138,12 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) { NamedXContentRegistry.EMPTY, new DummyS3Service(configPath()), BlobStoreTestUtil.mockClusterService(), - new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + null, + null, + null, + null, + false ) { @Override protected void assertSnapshotOrGenericThread() { From 68ddf2442c57672ff65c72f39c36e66dcd2e4dd7 Mon Sep 17 00:00:00 2001 From: Sam Hobbs <118170270+Sam-ScottLogic@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:40:46 +0100 Subject: [PATCH 118/186] =?UTF-8?q?Added=20the=20addition=20of=20the=20ser?= =?UTF-8?q?ver=20version=20and=20distribution=20as=20a=20respon=E2=80=A6?= =?UTF-8?q?=20(#8084)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added server distribution and version into response header Signed-off-by: Sam Hobbs * Removed extra lines Signed-off-by: Sam Hobbs * Update server/src/main/java/org/opensearch/http/DefaultRestChannel.java Co-authored-by: Owais Kazi Signed-off-by: Sam Hobbs <118170270+Sam-ScottLogic@users.noreply.github.com> * Moved defining of server header string Signed-off-by: Sam Hobbs * Removed wildcard import Signed-off-by: Sam Hobbs * Applied spotless to files Signed-off-by: Sam Hobbs * Added to testHeadersSet test within the default rest channel test file to test that server version is returned Signed-off-by: Sam Hobbs * Ran spotless Signed-off-by: Sam Hobbs * Added comment Signed-off-by: Sam Hobbs * Created static string for reused server header title and name of application strings, also refactored server header to server version Signed-off-by: Sam Hobbs * Refactored hash map from server header to server version and updated test Signed-off-by: Sam Hobbs * Ran spotless Signed-off-by: Sam Hobbs * Moved server version field from global variable to local, defined sever string as static value Signed-off-by: Sam Hobbs * Added prefix to header title Signed-off-by: Sam Hobbs * Removed redundant serverVersion field and instead inlined creation of map passed into addCustomHeader method Signed-off-by: Sam Hobbs * Ran spotless Signed-off-by: Sam Hobbs * Removed redundant OPEN_SEARCH_NAME_VALUE Signed-off-by: Sam Hobbs * Update server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java Co-authored-by: Andriy Redko Signed-off-by: Sam Hobbs <118170270+Sam-ScottLogic@users.noreply.github.com> * Moved change log addition to 2.x unreleased and wrote message in imperative Signed-off-by: Sam Hobbs * Corrected name of fetched header in header test Signed-off-by: Sam Hobbs * Added server version header value as constant Signed-off-by: Sam Hobbs --------- Signed-off-by: Sam Hobbs Signed-off-by: Sam Hobbs <118170270+Sam-ScottLogic@users.noreply.github.com> Co-authored-by: Owais Kazi Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + .../java/org/opensearch/http/DefaultRestChannel.java | 11 ++++++++++- .../org/opensearch/http/DefaultRestChannelTests.java | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef26caadc48bf..d5c54a5c695a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) - [Search pipelines] Add Global Ignore_failure options for Processors ([#8373](https://github.com/opensearch-project/OpenSearch/pull/8373)) - Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) +- Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) - Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) - Introduce full support for Search Pipeline ([#8613](https://github.com/opensearch-project/OpenSearch/pull/8613)) diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index 75a2a68dd78fe..523a89e05caa3 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -32,6 +32,7 @@ package org.opensearch.http; +import org.opensearch.Build; import org.opensearch.action.ActionListener; import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesArray; @@ -48,7 +49,6 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.core.rest.RestStatus; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -69,6 +69,12 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann static final String CONTENT_TYPE = "content-type"; static final String CONTENT_LENGTH = "content-length"; static final String SET_COOKIE = "set-cookie"; + static final String SERVER_VERSION = "X-OpenSearch-Version"; + static final String SERVER_VERSION_VALUE = "OpenSearch/" + + Build.CURRENT.getQualifiedVersion() + + " (" + + Build.CURRENT.getDistribution() + + ")"; private final HttpRequest httpRequest; private final BigArrays bigArrays; @@ -76,6 +82,7 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann private final ThreadContext threadContext; private final HttpChannel httpChannel; private final CorsHandler corsHandler; + private final Map> SERVER_VERSION_HEADER = Map.of(SERVER_VERSION, List.of(SERVER_VERSION_VALUE)); @Nullable private final HttpTracer tracerLog; @@ -147,6 +154,8 @@ public void sendResponse(RestResponse restResponse) { addCustomHeaders(httpResponse, restResponse.getHeaders()); addCustomHeaders(httpResponse, threadContext.getResponseHeaders()); + addCustomHeaders(httpResponse, SERVER_VERSION_HEADER); + // If our response doesn't specify a content-type header, set one setHeaderField(httpResponse, CONTENT_TYPE, restResponse.contentType(), false); // If our response has no content-length, calculate and set one diff --git a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java index 0338c9009ab86..1ffe0538edaad 100644 --- a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java +++ b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java @@ -32,6 +32,7 @@ package org.opensearch.http; +import org.opensearch.Build; import org.opensearch.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -189,6 +190,10 @@ public void testHeadersSet() { assertEquals("abc", headers.get(Task.X_OPAQUE_ID).get(0)); assertEquals(Integer.toString(resp.content().length()), headers.get(DefaultRestChannel.CONTENT_LENGTH).get(0)); assertEquals(resp.contentType(), headers.get(DefaultRestChannel.CONTENT_TYPE).get(0)); + assertEquals( + "OpenSearch/" + Build.CURRENT.getQualifiedVersion() + " (" + Build.CURRENT.getDistribution() + ")", + headers.get("X-OpenSearch-Version").get(0) + ); } public void testCookiesSet() { From 064f265362f6ae8296bfeed09aaaf48be86e6a4d Mon Sep 17 00:00:00 2001 From: Sorabh Date: Thu, 13 Jul 2023 10:08:52 -0700 Subject: [PATCH 119/186] CardinalityIT/NestedIT test failures with concurrent search enabled and AssertingCodec (#8303) * CardinalityIT/NestedIT test failures with concurrent search enabled and AssertingCodec The tests were failing because during the concurrent segment search for each slice the codec producers for the leafs were initialized by the slice thread. Later in reduce phase, the post collection happens over those codec producers on the search thread. With AssertingCodec it verifies that all access is done by the same thread causing the failures Signed-off-by: Sorabh Hamirwasia * Address review comments Signed-off-by: Sorabh Hamirwasia --------- Signed-off-by: Sorabh Hamirwasia --- CHANGELOG.md | 1 + .../search/DefaultSearchContext.java | 13 ++- .../AggregationCollectorManager.java | 26 +---- .../BucketCollectorProcessor.java | 105 ++++++++++++++++++ .../ConcurrentAggregationProcessor.java | 6 +- .../DefaultAggregationProcessor.java | 4 + .../search/internal/ContextIndexSearcher.java | 1 + .../internal/FilteredSearchContext.java | 11 ++ .../search/internal/SearchContext.java | 22 ++++ .../search/SearchCancellationTests.java | 13 ++- .../internal/ContextIndexSearcherTests.java | 1 + .../profile/query/QueryProfilerTests.java | 1 + .../search/query/QueryPhaseTests.java | 2 + .../search/query/QueryProfilePhaseTests.java | 2 + .../aggregations/AggregatorTestCase.java | 1 + .../opensearch/test/TestSearchContext.java | 12 ++ 16 files changed, 192 insertions(+), 29 deletions(-) create mode 100644 server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c54a5c695a8..267cb83d53e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,6 +174,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) - Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) - Add wrapper tracer implementation ([#8565](https://github.com/opensearch-project/OpenSearch/pull/8565)) +- Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index e83bfe8486904..ee29d6bfe2b62 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -65,6 +65,7 @@ import org.opensearch.index.search.NestedHelper; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.search.aggregations.BucketCollectorProcessor; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.builder.SearchSourceBuilder; @@ -176,7 +177,7 @@ final class DefaultSearchContext extends SearchContext { private SuggestionSearchContext suggest; private List rescore; private Profilers profilers; - + private BucketCollectorProcessor bucketCollectorProcessor = NO_OP_BUCKET_COLLECTOR_PROCESSOR; private final Map searchExtBuilders = new HashMap<>(); private final Map, CollectorManager> queryCollectorManagers = new HashMap<>(); private final QueryShardContext queryShardContext; @@ -919,4 +920,14 @@ public ReaderContext readerContext() { public InternalAggregation.ReduceContext partial() { return requestToAggReduceContextBuilder.apply(request.source()).forPartialReduction(); } + + @Override + public void setBucketCollectorProcessor(BucketCollectorProcessor bucketCollectorProcessor) { + this.bucketCollectorProcessor = bucketCollectorProcessor; + } + + @Override + public BucketCollectorProcessor bucketCollectorProcessor() { + return bucketCollectorProcessor; + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java index 03519b335bbea..0b36fc8b0cc5a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java @@ -17,11 +17,8 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Deque; -import java.util.LinkedList; import java.util.List; /** @@ -53,31 +50,12 @@ public Collector newCollector() throws IOException { @Override public ReduceableSearchResult reduce(Collection collectors) throws IOException { - List aggregators = new ArrayList<>(); - - final Deque allCollectors = new LinkedList<>(collectors); - while (!allCollectors.isEmpty()) { - final Collector currentCollector = allCollectors.pop(); - if (currentCollector instanceof Aggregator) { - aggregators.add((Aggregator) currentCollector); - } else if (currentCollector instanceof InternalProfileCollector) { - if (((InternalProfileCollector) currentCollector).getCollector() instanceof Aggregator) { - aggregators.add((Aggregator) ((InternalProfileCollector) currentCollector).getCollector()); - } else if (((InternalProfileCollector) currentCollector).getCollector() instanceof MultiBucketCollector) { - allCollectors.addAll( - Arrays.asList(((MultiBucketCollector) ((InternalProfileCollector) currentCollector).getCollector()).getCollectors()) - ); - } - } else if (currentCollector instanceof MultiBucketCollector) { - allCollectors.addAll(Arrays.asList(((MultiBucketCollector) currentCollector).getCollectors())); - } - } - + final List aggregators = context.bucketCollectorProcessor().toAggregators(collectors); final List internals = new ArrayList<>(aggregators.size()); context.aggregations().resetBucketMultiConsumer(); for (Aggregator aggregator : aggregators) { try { - aggregator.postCollection(); + // post collection is called in ContextIndexSearcher after search on leaves are completed internals.add(aggregator.buildTopLevel()); } catch (IOException e) { throw new AggregationExecutionException("Failed to build aggregation [" + aggregator.name() + "]", e); diff --git a/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java b/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java new file mode 100644 index 0000000000000..352ecf8bc94ad --- /dev/null +++ b/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations; + +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.MultiCollector; +import org.opensearch.common.lucene.MinimumScoreCollector; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.profile.query.InternalProfileCollector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +/** + * Processor to perform collector level processing specific to {@link BucketCollector} in different stages like: a) PostCollection + * after search on each leaf is completed and b) process the collectors to perform reduce after collection is completed + */ +public class BucketCollectorProcessor { + + /** + * Performs {@link BucketCollector#postCollection()} on all the {@link BucketCollector} in the given {@link Collector} collector tree + * after the collection of documents on a leaf is completed. This method will be called by different slice threads on its own collector + * tree instance in case of concurrent segment search such that postCollection happens on the same slice thread which initialize and + * perform collection of the documents for a leaf segment. For sequential search case, there is always a single search thread which + * performs both collection and postCollection on {@link BucketCollector}. + *

    + * This was originally done in {@link org.opensearch.search.aggregations.AggregationProcessor#postProcess(SearchContext)}. But with + * concurrent segment search path this needs to be performed here. There are AssertingCodecs in lucene which validates that the + * DocValues created for a field is always used by the same thread for a request. In concurrent segment search case, the DocValues + * gets initialized on different threads for different segments (or slices). Whereas the postProcess happens as part of reduce phase + * and is performed on the separate thread which is from search threadpool and not from slice threadpool. So two different threads + * performs the access on the DocValues causing the AssertingCodec to fail. From functionality perspective, there is no issue as + * DocValues for each segment is always accessed by a single thread at a time but those threads may be different (e.g. slice thread + * during collection and then search thread during reduce) + *

    + *

    + * NOTE: We can evaluate and deprecate this postCollection processing once lucene release the changes described in the + * issue-12375. With this new change we should be able to implement + * {@link BucketCollector#postCollection()} functionality using the lucene interface directly such that postCollection gets called + * from the slice thread by lucene itself + *

    + * @param collectorTree collector tree used by calling thread + */ + public void processPostCollection(Collector collectorTree) throws IOException { + final Queue collectors = new LinkedList<>(); + collectors.offer(collectorTree); + while (!collectors.isEmpty()) { + Collector currentCollector = collectors.poll(); + if (currentCollector instanceof InternalProfileCollector) { + collectors.offer(((InternalProfileCollector) currentCollector).getCollector()); + } else if (currentCollector instanceof MinimumScoreCollector) { + collectors.offer(((MinimumScoreCollector) currentCollector).getCollector()); + } else if (currentCollector instanceof MultiCollector) { + for (Collector innerCollector : ((MultiCollector) currentCollector).getCollectors()) { + collectors.offer(innerCollector); + } + } else if (currentCollector instanceof BucketCollector) { + ((BucketCollector) currentCollector).postCollection(); + } + } + } + + /** + * Unwraps the input collection of {@link Collector} to get the list of the {@link Aggregator} used by different slice threads. The + * input is expected to contain the collectors related to Aggregations only as that is passed to {@link AggregationCollectorManager} + * during the reduce phase. This list of {@link Aggregator} is used to create {@link InternalAggregation} and optionally perform + * reduce at shard level before returning response to coordinator + * @param collectors collection of aggregation collectors to reduce + * @return list of unwrapped {@link Aggregator} + */ + public List toAggregators(Collection collectors) { + List aggregators = new ArrayList<>(); + + final Deque allCollectors = new LinkedList<>(collectors); + while (!allCollectors.isEmpty()) { + final Collector currentCollector = allCollectors.pop(); + if (currentCollector instanceof Aggregator) { + aggregators.add((Aggregator) currentCollector); + } else if (currentCollector instanceof InternalProfileCollector) { + if (((InternalProfileCollector) currentCollector).getCollector() instanceof Aggregator) { + aggregators.add((Aggregator) ((InternalProfileCollector) currentCollector).getCollector()); + } else if (((InternalProfileCollector) currentCollector).getCollector() instanceof MultiBucketCollector) { + allCollectors.addAll( + Arrays.asList(((MultiBucketCollector) ((InternalProfileCollector) currentCollector).getCollector()).getCollectors()) + ); + } + } else if (currentCollector instanceof MultiBucketCollector) { + allCollectors.addAll(Arrays.asList(((MultiBucketCollector) currentCollector).getCollectors())); + } + } + return aggregators; + } +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/ConcurrentAggregationProcessor.java b/server/src/main/java/org/opensearch/search/aggregations/ConcurrentAggregationProcessor.java index 592fb8cc6e674..336ad8739eb41 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/ConcurrentAggregationProcessor.java +++ b/server/src/main/java/org/opensearch/search/aggregations/ConcurrentAggregationProcessor.java @@ -28,12 +28,16 @@ * avoid the increase in aggregation result sets returned by each shard to coordinator where final reduce happens for results received from * all the shards */ -public class ConcurrentAggregationProcessor extends DefaultAggregationProcessor { +public class ConcurrentAggregationProcessor implements AggregationProcessor { + + private final BucketCollectorProcessor bucketCollectorProcessor = new BucketCollectorProcessor(); @Override public void preProcess(SearchContext context) { try { if (context.aggregations() != null) { + // update the bucket collector process as there is aggregation in the request + context.setBucketCollectorProcessor(bucketCollectorProcessor); if (context.aggregations().factories().hasNonGlobalAggregator()) { context.queryCollectorManagers().put(NonGlobalAggCollectorManager.class, new NonGlobalAggCollectorManager(context)); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/DefaultAggregationProcessor.java b/server/src/main/java/org/opensearch/search/aggregations/DefaultAggregationProcessor.java index 05aa4a9acb270..24b05ebcf3a61 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/DefaultAggregationProcessor.java +++ b/server/src/main/java/org/opensearch/search/aggregations/DefaultAggregationProcessor.java @@ -24,10 +24,14 @@ */ public class DefaultAggregationProcessor implements AggregationProcessor { + private final BucketCollectorProcessor bucketCollectorProcessor = new BucketCollectorProcessor(); + @Override public void preProcess(SearchContext context) { try { if (context.aggregations() != null) { + // update the bucket collector process as there is aggregation in the request + context.setBucketCollectorProcessor(bucketCollectorProcessor); if (context.aggregations().factories().hasNonGlobalAggregator()) { context.queryCollectorManagers() .put(NonGlobalAggCollectorManager.class, new NonGlobalAggCollectorManagerWithSingleCollector(context)); diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index 2f1c5475c9cf6..e3ca932eb4699 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -276,6 +276,7 @@ protected void search(List leaves, Weight weight, Collector c searchLeaf(leaves.get(i), weight, collector); } } + searchContext.bucketCollectorProcessor().processPostCollection(collector); } /** diff --git a/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java index 9bfc0e8b6fea5..790d2ed5ee4b7 100644 --- a/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java @@ -50,6 +50,7 @@ import org.opensearch.index.similarity.SimilarityService; import org.opensearch.search.SearchExtBuilder; import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.aggregations.BucketCollectorProcessor; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.collapse.CollapseContext; @@ -548,4 +549,14 @@ public ReaderContext readerContext() { public InternalAggregation.ReduceContext partial() { return in.partial(); } + + @Override + public void setBucketCollectorProcessor(BucketCollectorProcessor bucketCollectorProcessor) { + in.setBucketCollectorProcessor(bucketCollectorProcessor); + } + + @Override + public BucketCollectorProcessor bucketCollectorProcessor() { + return in.bucketCollectorProcessor(); + } } diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index 319a5624bbf56..fd02ba2ba12bb 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -53,6 +53,8 @@ import org.opensearch.search.RescoreDocIds; import org.opensearch.search.SearchExtBuilder; import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.BucketCollectorProcessor; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.collapse.CollapseContext; @@ -73,6 +75,7 @@ import org.opensearch.search.sort.SortAndFormats; import org.opensearch.search.suggest.SuggestionSearchContext; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -94,6 +97,20 @@ public abstract class SearchContext implements Releasable { public static final int TRACK_TOTAL_HITS_DISABLED = -1; public static final int DEFAULT_TRACK_TOTAL_HITS_UP_TO = 10000; + // no-op bucket collector processor + public static final BucketCollectorProcessor NO_OP_BUCKET_COLLECTOR_PROCESSOR = new BucketCollectorProcessor() { + @Override + public void processPostCollection(Collector collectorTree) { + // do nothing as there is no aggregation collector + } + + @Override + public List toAggregators(Collection collectors) { + // should not be called when there is no aggregation collector + throw new IllegalStateException("Unexpected toAggregators call on NO_OP_BUCKET_COLLECTOR_PROCESSOR"); + } + }; + private final List releasables = new CopyOnWriteArrayList<>(); private final AtomicBoolean closed = new AtomicBoolean(false); private InnerHitsContext innerHitsContext; @@ -449,4 +466,9 @@ public String toString() { public abstract ReaderContext readerContext(); public abstract InternalAggregation.ReduceContext partial(); + + // processor used for bucket collectors + public abstract void setBucketCollectorProcessor(BucketCollectorProcessor bucketCollectorProcessor); + + public abstract BucketCollectorProcessor bucketCollectorProcessor(); } diff --git a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java index e67123bf2c51e..011723da36a30 100644 --- a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java @@ -48,6 +48,7 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; +import org.junit.Before; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.IndexShard; import org.opensearch.search.internal.ContextIndexSearcher; @@ -71,6 +72,7 @@ public class SearchCancellationTests extends OpenSearchTestCase { private static Directory dir; private static IndexReader reader; + private SearchContext searchContext; @BeforeClass public static void setup() throws IOException { @@ -106,6 +108,12 @@ public static void cleanup() throws IOException { reader = null; } + @Before + public void testSetup() { + searchContext = mock(SearchContext.class); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); + } + public void testAddingCancellationActions() throws IOException { ContextIndexSearcher searcher = new ContextIndexSearcher( reader, @@ -114,7 +122,7 @@ public void testAddingCancellationActions() throws IOException { IndexSearcher.getDefaultQueryCachingPolicy(), true, null, - mock(SearchContext.class) + searchContext ); NullPointerException npe = expectThrows(NullPointerException.class, () -> searcher.addQueryCancellation(null)); assertEquals("cancellation runnable should not be null", npe.getMessage()); @@ -128,7 +136,6 @@ public void testAddingCancellationActions() throws IOException { public void testCancellableCollector() throws IOException { TotalHitCountCollector collector1 = new TotalHitCountCollector(); Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); }; - SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); ContextIndexSearcher searcher = new ContextIndexSearcher( @@ -167,7 +174,7 @@ public void testExitableDirectoryReader() throws IOException { IndexSearcher.getDefaultQueryCachingPolicy(), true, null, - mock(SearchContext.class) + searchContext ); searcher.addQueryCancellation(cancellation); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton()); diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index a2c4288e081b3..f3907355ac6ec 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -259,6 +259,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); ContextIndexSearcher searcher = new ContextIndexSearcher( filteredReader, IndexSearcher.getDefaultSimilarity(), diff --git a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java index f0f692e15f066..528d65bcc5ef2 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java @@ -120,6 +120,7 @@ public void setUp() throws Exception { SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); searcher = new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index cf4d7798056c4..61b78905334ec 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -1207,6 +1207,7 @@ private static ContextIndexSearcher newContextSearcher(IndexReader reader, Execu SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), @@ -1223,6 +1224,7 @@ private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexRead SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 9c27783263c5a..6d30d7993c850 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1427,6 +1427,7 @@ private static ContextIndexSearcher newContextSearcher(IndexReader reader, Execu SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), @@ -1443,6 +1444,7 @@ private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexRead SearchContext searchContext = mock(SearchContext.class); IndexShard indexShard = mock(IndexShard.class); when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); return new ContextIndexSearcher( reader, IndexSearcher.getDefaultSimilarity(), diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index 3cddf4659fea9..60d337599771c 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -349,6 +349,7 @@ public boolean shouldCache(Query query) { when(searchContext.indexShard()).thenReturn(indexShard); when(searchContext.aggregations()).thenReturn(new SearchContextAggregations(AggregatorFactories.EMPTY, bucketConsumer)); when(searchContext.query()).thenReturn(query); + when(searchContext.bucketCollectorProcessor()).thenReturn(new BucketCollectorProcessor()); /* * Always use the circuit breaking big arrays instance so that the CircuitBreakerService * we're passed gets a chance to break. diff --git a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java index 60425077b14a0..694f88d944f71 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java @@ -52,6 +52,7 @@ import org.opensearch.index.similarity.SimilarityService; import org.opensearch.search.SearchExtBuilder; import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.aggregations.BucketCollectorProcessor; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.collapse.CollapseContext; @@ -116,6 +117,7 @@ public class TestSearchContext extends SearchContext { private Profilers profilers; private CollapseContext collapse; protected boolean concurrentSegmentSearchEnabled; + private BucketCollectorProcessor bucketCollectorProcessor = NO_OP_BUCKET_COLLECTOR_PROCESSOR; /** * Sets the concurrent segment search enabled field @@ -661,6 +663,16 @@ public InternalAggregation.ReduceContext partial() { return InternalAggregationTestCase.emptyReduceContextBuilder().forPartialReduction(); } + @Override + public void setBucketCollectorProcessor(BucketCollectorProcessor bucketCollectorProcessor) { + this.bucketCollectorProcessor = bucketCollectorProcessor; + } + + @Override + public BucketCollectorProcessor bucketCollectorProcessor() { + return bucketCollectorProcessor; + } + /** * Clean the query results by consuming all of it */ From 3254efd7decd0d9a97caa69982286b1b90198585 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 13 Jul 2023 13:19:22 -0500 Subject: [PATCH 120/186] Create 2.9 release notes and clear out changelog (#8682) * Create 2.9 release notes and clear out changelog Signed-off-by: Andrew Ross * Move changes from 3.0 that were backported Signed-off-by: Andrew Ross --------- Signed-off-by: Andrew Ross --- CHANGELOG.md | 102 +--------------- .../opensearch.release-notes-2.9.0.md | 109 ++++++++++++++++++ 2 files changed, 111 insertions(+), 100 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.9.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 267cb83d53e01..e10043b15f092 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) - Add support for ignoring missing Javadoc on generated code using annotation ([#7604](https://github.com/opensearch-project/OpenSearch/pull/7604)) -- Add partial results support for concurrent segment search ([#8306](https://github.com/opensearch-project/OpenSearch/pull/8306)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 @@ -49,10 +48,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919)) -- Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909)) -- Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) -- Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) -- Add wrapper tracer implementation ### Deprecated @@ -77,119 +72,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460)) - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230)) -- Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541)) -- Adds log4j configuration for telemetry LogSpanExporter ([#8393](https://github.com/opensearch-project/OpenSearch/pull/8393)) -- Fix painless casting bug, which crashes the OpenSearch process ([#8315](https://github.com/opensearch-project/OpenSearch/pull/8315)) ### Security ## [Unreleased 2.x] ### Added -- [SearchPipeline] Add new search pipeline processor type, SearchPhaseResultsProcessor, that can modify the result of one search phase before starting the next phase.([#7283](https://github.com/opensearch-project/OpenSearch/pull/7283)) -- Add task cancellation monitoring service ([#7642](https://github.com/opensearch-project/OpenSearch/pull/7642)) -- Add TokenManager Interface ([#7452](https://github.com/opensearch-project/OpenSearch/pull/7452)) -- Add Remote store as a segment replication source ([#7653](https://github.com/opensearch-project/OpenSearch/pull/7653)) -- Implement concurrent aggregations support without profile option ([#7514](https://github.com/opensearch-project/OpenSearch/pull/7514)) -- Add dynamic index and cluster setting for concurrent segment search ([#7956](https://github.com/opensearch-project/OpenSearch/pull/7956)) -- Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967)) -- [Search pipelines] Added search pipelines output to node stats ([#8053](https://github.com/opensearch-project/OpenSearch/pull/8053)) -- Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020)) -- Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038)) -- Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) -- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) -- Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) -- Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) -- [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) -- Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) -- Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) -- Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) -- [Search pipelines] Add Global Ignore_failure options for Processors ([#8373](https://github.com/opensearch-project/OpenSearch/pull/8373)) -- Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) - Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) -- Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) -- Introduce full support for Search Pipeline ([#8613](https://github.com/opensearch-project/OpenSearch/pull/8613)) +- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) ### Dependencies -- Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) -- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401, #8581) -- Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812) -- Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810) -- Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896) -- Bump `jackson` from 2.15.1 to 2.15.2 ([#7897](https://github.com/opensearch-project/OpenSearch/pull/7897)) -- Add `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996)) -- Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901)) -- Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) -- Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) -- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.85 ([7968], #8255) -- Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068)) -- Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre (#8009) -- Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 (#8014, #8013, #8010) -- Bump `com.diffplug.spotless` from 6.18.0 to 6.19.0 (#8007) -- Bump `'com.azure:azure-storage-blob` to 12.22.2 from 12.21.1 ([#8043](https://github.com/opensearch-project/OpenSearch/pull/8043)) -- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015, #8254) -- Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre ([#8011](https://github.com/opensearch-project/OpenSearch/pull/8011), [#8012](https://github.com/opensearch-project/OpenSearch/pull/8012), [#8107](https://github.com/opensearch-project/OpenSearch/pull/8107)) -- Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016)) -- Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122)) -- Bump `com.networknt:json-schema-validator` from 1.0.83 to 1.0.84 (#8141) -- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.5 (#8139, #8568) -- Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140)) -- Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181)) -- Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) -- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257) -- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256) -- [Upgrade] Lucene 9.7.0 release (#8272) -- Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209)) -- Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276)) -- Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434)) -- Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) -- Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) -- Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) -- Bump `net.minidev:json-smart` from 2.4.11 to 2.5.0 (#8575, #8576) -- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) -- Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) -- Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) -- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 ([#8585](https://github.com/opensearch-project/OpenSearch/pull/8585)) - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) ### Changed -- Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) -- Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681)) -- Add min, max, average and thread info to resource stats in tasks API ([#7673](https://github.com/opensearch-project/OpenSearch/pull/7673)) -- Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996)) -- Change `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` to `aws.ec2MetadataServiceEndpoint` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) -- Change `com.amazonaws.sdk.stsEndpointOverride` to `aws.stsEndpointOverride` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) -- Add new query profile collector fields with concurrent search execution ([#7898](https://github.com/opensearch-project/OpenSearch/pull/7898)) -- Align range and default value for deletes_pct_allowed in merge policy ([#7730](https://github.com/opensearch-project/OpenSearch/pull/7730)) -- Rename QueryPhase actors like Suggest, Rescore to be processors rather than phase ([#8025](https://github.com/opensearch-project/OpenSearch/pull/8025)) -- Compress and cache cluster state during validate join request ([#7321](https://github.com/opensearch-project/OpenSearch/pull/7321)) -- [Snapshot Interop] Add Changes in Create Snapshot Flow for remote store interoperability. ([#7118](https://github.com/opensearch-project/OpenSearch/pull/7118)) -- Allow insecure string settings to warn-log usage and advise to migration of a newer secure variant ([#5496](https://github.com/opensearch-project/OpenSearch/pull/5496)) -- Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652)) -- Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963)) -- Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) -- Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) -- [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) -- Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) -- Optimize Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853)) -- Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) -- Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) -- Add wrapper tracer implementation ([#8565](https://github.com/opensearch-project/OpenSearch/pull/8565)) - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) +- Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) ### Deprecated ### Removed -- Remove `COMPRESSOR` variable from `CompressorFactory` and use `DEFLATE_COMPRESSOR` instead ([7907](https://github.com/opensearch-project/OpenSearch/pull/7907)) -- Remove concurrency based minimum file cache size restriction ([#8294](https://github.com/opensearch-project/OpenSearch/pull/8294)) ### Fixed -- Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924 -- Enforce 512 byte document ID limit in bulk updates ([#8039](https://github.com/opensearch-project/OpenSearch/pull/8039)) -- With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125)) -- Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591)) -- Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318)) -- Precise system clock time in MasterService debug logs ([#7902](https://github.com/opensearch-project/OpenSearch/pull/7902)) -- Improve indexing performance for flat_object type ([#7855](https://github.com/opensearch-project/OpenSearch/pull/7855)) ### Security diff --git a/release-notes/opensearch.release-notes-2.9.0.md b/release-notes/opensearch.release-notes-2.9.0.md new file mode 100644 index 0000000000000..76d2d9f4593d6 --- /dev/null +++ b/release-notes/opensearch.release-notes-2.9.0.md @@ -0,0 +1,109 @@ +## 2023-07-13 Version 2.9.0 Release Notes + +## [2.9] + +### Added +- [SearchPipeline] Add new search pipeline processor type, SearchPhaseResultsProcessor, that can modify the result of one search phase before starting the next phase.([#7283](https://github.com/opensearch-project/OpenSearch/pull/7283)) +- Add task cancellation monitoring service ([#7642](https://github.com/opensearch-project/OpenSearch/pull/7642)) +- Add TokenManager Interface ([#7452](https://github.com/opensearch-project/OpenSearch/pull/7452)) +- Add Remote store as a segment replication source ([#7653](https://github.com/opensearch-project/OpenSearch/pull/7653)) +- Implement concurrent aggregations support without profile option ([#7514](https://github.com/opensearch-project/OpenSearch/pull/7514)) +- Add dynamic index and cluster setting for concurrent segment search ([#7956](https://github.com/opensearch-project/OpenSearch/pull/7956)) +- Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967)) +- [Search pipelines] Added search pipelines output to node stats ([#8053](https://github.com/opensearch-project/OpenSearch/pull/8053)) +- Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020)) +- Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038)) +- Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029) +- Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543)) +- Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168)) +- [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414)) +- Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545)) +- Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957)) +- Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291) +- [Search pipelines] Add Global Ignore_failure options for Processors ([#8373](https://github.com/opensearch-project/OpenSearch/pull/8373)) +- Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997)) +- Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) +- Introduce full support for Search Pipeline ([#8613](https://github.com/opensearch-project/OpenSearch/pull/8613)) +- Add partial results support for concurrent segment search ([#8306](https://github.com/opensearch-project/OpenSearch/pull/8306)) + +### Dependencies +- Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) +- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401, #8581) +- Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812) +- Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810) +- Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896) +- Bump `jackson` from 2.15.1 to 2.15.2 ([#7897](https://github.com/opensearch-project/OpenSearch/pull/7897)) +- Add `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996)) +- Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901)) +- Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) +- Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) +- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.85 ([7968], #8255) +- Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068)) +- Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre (#8009) +- Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 (#8014, #8013, #8010) +- Bump `com.diffplug.spotless` from 6.18.0 to 6.19.0 (#8007) +- Bump `'com.azure:azure-storage-blob` to 12.22.2 from 12.21.1 ([#8043](https://github.com/opensearch-project/OpenSearch/pull/8043)) +- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015, #8254) +- Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre ([#8011](https://github.com/opensearch-project/OpenSearch/pull/8011), [#8012](https://github.com/opensearch-project/OpenSearch/pull/8012), [#8107](https://github.com/opensearch-project/OpenSearch/pull/8107)) +- Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016)) +- Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122)) +- Bump `com.networknt:json-schema-validator` from 1.0.83 to 1.0.84 (#8141) +- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.5 (#8139, #8568) +- Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140)) +- Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181)) +- Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191)) +- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257) +- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256) +- [Upgrade] Lucene 9.7.0 release (#8272) +- Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209)) +- Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276)) +- Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434)) +- Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403) +- Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405) +- Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572) +- Bump `net.minidev:json-smart` from 2.4.11 to 2.5.0 (#8575, #8576) +- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 (#8577, #8571) +- Bump `com.networknt:json-schema-validator` from 1.0.85 to 1.0.86 ([#8573](https://github.com/opensearch-project/OpenSearch/pull/8573)) +- Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586)) +- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 ([#8585](https://github.com/opensearch-project/OpenSearch/pull/8585)) + +### Changed +- Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836)) +- Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681)) +- Add min, max, average and thread info to resource stats in tasks API ([#7673](https://github.com/opensearch-project/OpenSearch/pull/7673)) +- Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996)) +- Change `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` to `aws.ec2MetadataServiceEndpoint` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) +- Change `com.amazonaws.sdk.stsEndpointOverride` to `aws.stsEndpointOverride` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/)) +- Add new query profile collector fields with concurrent search execution ([#7898](https://github.com/opensearch-project/OpenSearch/pull/7898)) +- Align range and default value for deletes_pct_allowed in merge policy ([#7730](https://github.com/opensearch-project/OpenSearch/pull/7730)) +- Rename QueryPhase actors like Suggest, Rescore to be processors rather than phase ([#8025](https://github.com/opensearch-project/OpenSearch/pull/8025)) +- Compress and cache cluster state during validate join request ([#7321](https://github.com/opensearch-project/OpenSearch/pull/7321)) +- [Snapshot Interop] Add Changes in Create Snapshot Flow for remote store interoperability. ([#7118](https://github.com/opensearch-project/OpenSearch/pull/7118)) +- Allow insecure string settings to warn-log usage and advise to migration of a newer secure variant ([#5496](https://github.com/opensearch-project/OpenSearch/pull/5496)) +- Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652)) +- Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963)) +- Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908)) +- Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471)) +- [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164)) +- Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312)) +- Optimize Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853)) +- Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) +- Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411)) +- Add wrapper tracer implementation ([#8565](https://github.com/opensearch-project/OpenSearch/pull/8565)) +- Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909)) + +### Removed +- Remove `COMPRESSOR` variable from `CompressorFactory` and use `DEFLATE_COMPRESSOR` instead ([7907](https://github.com/opensearch-project/OpenSearch/pull/7907)) +- Remove concurrency based minimum file cache size restriction ([#8294](https://github.com/opensearch-project/OpenSearch/pull/8294)) + +### Fixed +- Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924 +- Enforce 512 byte document ID limit in bulk updates ([#8039](https://github.com/opensearch-project/OpenSearch/pull/8039)) +- With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125)) +- Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591)) +- Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318)) +- Precise system clock time in MasterService debug logs ([#7902](https://github.com/opensearch-project/OpenSearch/pull/7902)) +- Improve indexing performance for flat_object type ([#7855](https://github.com/opensearch-project/OpenSearch/pull/7855)) +- Adds log4j configuration for telemetry LogSpanExporter ([#8393](https://github.com/opensearch-project/OpenSearch/pull/8393)) +- Fix painless casting bug, which crashes the OpenSearch process ([#8315](https://github.com/opensearch-project/OpenSearch/pull/8315)) +- Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541)) From ab68d3e0742e43837f5754e9e7784ead95f37643 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Thu, 13 Jul 2023 11:59:22 -0700 Subject: [PATCH 121/186] Remove unused extension directory (#8674) Signed-off-by: Owais Kazi --- .../extensions/ExtensionsManagerTests.java | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 415ed4567163c..41b841aec7be8 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -23,9 +23,7 @@ import java.io.IOException; import java.net.InetAddress; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -95,33 +93,11 @@ public class ExtensionsManagerTests extends OpenSearchTestCase { private Setting customSetting = Setting.simpleString("custom_extension_setting", "none", Property.ExtensionScope); private NodeClient client; private MockNioTransport transport; - private Path extensionDir; private final ThreadPool threadPool = new TestThreadPool(ExtensionsManagerTests.class.getSimpleName()); private final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); - private final List extensionsYmlLines = Arrays.asList( - "extensions:", - " - name: firstExtension", - " uniqueId: uniqueid1", - " hostAddress: '127.0.0.0'", - " port: '9300'", - " version: '0.0.7'", - " opensearchVersion: '3.0.0'", - " minimumCompatibleVersion: '3.0.0'", - " custom_extension_setting: 'custom_setting'", - " - name: secondExtension", - " uniqueId: 'uniqueid2'", - " hostAddress: '127.0.0.1'", - " port: '9301'", - " version: '3.14.16'", - " opensearchVersion: '2.0.0'", - " minimumCompatibleVersion: '2.0.0'", - " dependencies:", - " - uniqueId: 'uniqueid0'", - " version: '2.0.0'" - ); private DiscoveryExtensionNode extensionNode; @@ -178,8 +154,6 @@ public List> getExtensionSettings() { settingsModule = new SettingsModule(Settings.EMPTY, emptyList(), emptyList(), emptySet()); clusterService = createClusterService(threadPool); - extensionDir = createTempDir(); - extensionNode = new DiscoveryExtensionNode( "firstExtension", "uniqueid1", From 6a19660ba203cd5ea0f13fae9844044ffd58df90 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 13 Jul 2023 15:11:58 -0500 Subject: [PATCH 122/186] Remove CHANGELOG entries that have been backported (#8686) The changes removed here from `[Unreleased 3.0]` have all been backported in previous releases. Signed-off-by: Andrew Ross --- CHANGELOG.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e10043b15f092..d3dfe121a8522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636)) - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) -- Add support for ignoring missing Javadoc on generated code using annotation ([#7604](https://github.com/opensearch-project/OpenSearch/pull/7604)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 @@ -45,9 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638)) - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459)) - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) -- Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) -- Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919)) ### Deprecated @@ -70,8 +67,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) -- Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460)) -- Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230)) ### Security From e8f1c30eb4fb8afa0176f721ebc6762d150b35cf Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:48:46 -0700 Subject: [PATCH 123/186] Changing search shard failure logging to trace from debug (#8470) * Changing shard failure logging to trace from debug * Moving the exception to trace logs Signed-off-by: Sagar Upadhyaya --- .../search/AbstractSearchAsyncAction.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index c67b56fed159e..152e6f56668f2 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -455,15 +455,27 @@ private void onShardFailure(final int shardIndex, @Nullable SearchShardTarget sh .findNext(shardIt, clusterState, e, () -> totalOps.incrementAndGet()); final boolean lastShard = nextShard == null; - logger.debug( - () -> new ParameterizedMessage( - "{}: Failed to execute [{}] lastShard [{}]", - shard != null ? shard : shardIt.shardId(), - request, - lastShard - ), - e - ); + if (logger.isTraceEnabled()) { + logger.trace( + () -> new ParameterizedMessage( + "{}: Failed to execute [{}] lastShard [{}]", + shard != null ? shard : shardIt.shardId(), + request, + lastShard + ), + e + ); + } else { + // Log the message without an exception. + logger.debug( + new ParameterizedMessage( + "{}: Failed to execute [{}] lastShard [{}]", + shard != null ? shard : shardIt.shardId(), + request, + lastShard + ) + ); + } if (lastShard) { onShardGroupFailure(shardIndex, shard, e); } From 7e8740f855586f48cc3bb76258f3bf883da47452 Mon Sep 17 00:00:00 2001 From: Ankit Kala Date: Fri, 14 Jul 2023 11:19:51 +0530 Subject: [PATCH 124/186] Handle replication cases where the remote store metadata file is not present (#8433) * Handle replication cases where the remote store metadata file is not present Signed-off-by: Ankit Kala Signed-off-by: Ashish Singh * Allow null metadata file only if shard is not started Signed-off-by: Ankit Kala --------- Signed-off-by: Ankit Kala Signed-off-by: Ashish Singh --- .../RemoteStoreReplicationSource.java | 12 ++++- .../RemoteStoreReplicationSourceTests.java | 49 ++++++++++++++++++- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java index b8941737a7c57..c5be7635782af 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java @@ -14,9 +14,11 @@ import org.apache.lucene.util.Version; import org.opensearch.action.ActionListener; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import java.util.Collections; @@ -53,8 +55,14 @@ public void getCheckpointMetadata( // TODO: Need to figure out a way to pass this information for segment metadata via remote store. final Version version = indexShard.getSegmentInfosSnapshot().get().getCommitLuceneVersion(); try { - metadataMap = remoteDirectory.readLatestMetadataFile() - .getMetadata() + RemoteSegmentMetadata mdFile = remoteDirectory.readLatestMetadataFile(); + // During initial recovery flow, the remote store might not have metadata as primary hasn't uploaded anything yet. + if (mdFile == null && indexShard.state().equals(IndexShardState.STARTED) == false) { + listener.onResponse(new CheckpointInfoResponse(checkpoint, Collections.emptyMap(), null)); + return; + } + assert mdFile != null : "Remote metadata file can't be null if shard is active " + indexShard.state(); + metadataMap = mdFile.getMetadata() .entrySet() .stream() .collect( diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java index 2209514e06438..d80d19e46e728 100644 --- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java @@ -17,6 +17,7 @@ import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.RemoteStoreRefreshListenerTests; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; @@ -42,6 +43,8 @@ public class RemoteStoreReplicationSourceTests extends OpenSearchIndexLevelRepli private IndexShard mockShard; + private Store remoteStore; + @Override public void setUp() throws Exception { super.setUp(); @@ -64,7 +67,7 @@ public void setUp() throws Exception { Store store = mock(Store.class); when(mockShard.store()).thenReturn(store); when(store.directory()).thenReturn(indexShard.store().directory()); - Store remoteStore = mock(Store.class); + remoteStore = mock(Store.class); when(mockShard.remoteStore()).thenReturn(remoteStore); RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()).getDelegate()) @@ -73,7 +76,6 @@ public void setUp() throws Exception { new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory) ); when(remoteStore.directory()).thenReturn(remoteStoreFilterDirectory); - replicationSource = new RemoteStoreReplicationSource(mockShard); } @@ -118,6 +120,49 @@ public void testGetCheckpointMetadataFailure() { }); } + public void testGetCheckpointMetadataEmpty() throws ExecutionException, InterruptedException, IOException { + when(mockShard.getSegmentInfosSnapshot()).thenReturn(indexShard.getSegmentInfosSnapshot()); + final ReplicationCheckpoint checkpoint = new ReplicationCheckpoint( + indexShard.shardId(), + PRIMARY_TERM, + SEGMENTS_GEN, + VERSION, + Codec.getDefault().getName() + ); + IndexShard emptyIndexShard = null; + try { + emptyIndexShard = newStartedShard( + true, + Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(), + new InternalEngineFactory() + ); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = + (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) emptyIndexShard.remoteStore().directory()).getDelegate()) + .getDelegate(); + FilterDirectory remoteStoreFilterDirectory = new RemoteStoreRefreshListenerTests.TestFilterDirectory( + new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory) + ); + when(remoteStore.directory()).thenReturn(remoteStoreFilterDirectory); + + final PlainActionFuture res = PlainActionFuture.newFuture(); + when(mockShard.state()).thenReturn(IndexShardState.RECOVERING); + // Recovering shard should just do a noop and return empty metadata map. + replicationSource.getCheckpointMetadata(REPLICATION_ID, checkpoint, res); + CheckpointInfoResponse response = res.get(); + assert (response.getCheckpoint().equals(checkpoint)); + assert (response.getMetadataMap().isEmpty()); + + when(mockShard.state()).thenReturn(IndexShardState.STARTED); + // Started shard should fail with assertion error. + expectThrows(AssertionError.class, () -> { + final PlainActionFuture res2 = PlainActionFuture.newFuture(); + replicationSource.getCheckpointMetadata(REPLICATION_ID, checkpoint, res2); + }); + } finally { + closeShards(emptyIndexShard); + } + } + public void testGetSegmentFiles() throws ExecutionException, InterruptedException { final ReplicationCheckpoint checkpoint = new ReplicationCheckpoint( indexShard.shardId(), From 675acdb4b92d862ccec3ebd10f390f333d81291f Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Fri, 14 Jul 2023 14:05:52 -0700 Subject: [PATCH 125/186] Fix RemoteStoreReplicationSourceTests.testGetCheckpointMetadataEmpty. (#8704) This change fixes RemoteStoreReplicationSourceTests.testGetCheckpointMetadataEmpty by properly setting replication type for on primary shards. Signed-off-by: Marc Handalian --- .../RemoteStoreReplicationSourceTests.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java index d80d19e46e728..deae4eaca19e7 100644 --- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java @@ -45,18 +45,16 @@ public class RemoteStoreReplicationSourceTests extends OpenSearchIndexLevelRepli private Store remoteStore; + private final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + @Override public void setUp() throws Exception { super.setUp(); - indexShard = newStartedShard( - true, - Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .build(), - new InternalEngineFactory() - ); + indexShard = newStartedShard(true, settings, new InternalEngineFactory()); indexDoc(indexShard, "_doc", "1"); indexDoc(indexShard, "_doc", "2"); @@ -133,7 +131,7 @@ public void testGetCheckpointMetadataEmpty() throws ExecutionException, Interrup try { emptyIndexShard = newStartedShard( true, - Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(), + settings, new InternalEngineFactory() ); RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = From ca74aac1482718a864676e27707bb77b7c168e6f Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Fri, 14 Jul 2023 16:44:50 -0700 Subject: [PATCH 126/186] Segment Replication - Allow search idle with no replicas (#8173) * Segment Replication - Allow shard idle when there are no replicas an index. Signed-off-by: Marc Handalian * Add warning when updating search.idle.after with Segment Replication enabled. Signed-off-by: Marc Handalian * PR cleanup. Signed-off-by: Marc Handalian * Add more tests. Signed-off-by: Marc Handalian * Remove unnecessary assertBusy Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../action/search/TransportSearchIT.java | 43 ++++++++++ .../org/opensearch/index/IndexSettings.java | 3 + .../opensearch/index/shard/IndexShard.java | 18 ++++- .../SegmentReplicationIndexShardTests.java | 79 +++++++++++++------ 4 files changed, 115 insertions(+), 28 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java index c22319271b4cf..afa5ac908c137 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java @@ -61,6 +61,7 @@ import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.core.rest.RestStatus; @@ -370,6 +371,48 @@ public void testSearchIdle() throws Exception { }); } + public void testSearchIdleWithSegmentReplication() { + int numOfReplicas = 1; + internalCluster().ensureAtLeastNumDataNodes(numOfReplicas + 1); + final Settings.Builder settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); + assertAcked(prepareCreate("test").setSettings(settings).setMapping("created_date", "type=date,format=yyyy-MM-dd")); + ensureGreen("test"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings( + Settings.builder() + .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMillis(randomIntBetween(50, 500))) + ) + ); + + for (String node : internalCluster().nodesInclude("test")) { + final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + for (IndexShard indexShard : indicesService.indexServiceSafe(resolveIndex("test"))) { + assertFalse(indexShard.isSearchIdleSupported()); + } + } + + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)) + ); + + for (String node : internalCluster().nodesInclude("test")) { + final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + for (IndexShard indexShard : indicesService.indexServiceSafe(resolveIndex("test"))) { + assertTrue(indexShard.isSearchIdleSupported()); + } + } + ; + } + public void testCircuitBreakerReduceFail() throws Exception { int numShards = randomIntBetween(1, 10); indexSomeDocs("test", numShards, numShards * 3); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 30c9efde19cdc..5b96b09a65cea 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -930,6 +930,9 @@ private void setSearchSegmentOrderReversed(boolean reversed) { } private void setSearchIdleAfter(TimeValue searchIdleAfter) { + if (this.replicationType == ReplicationType.SEGMENT && this.getNumberOfReplicas() > 0) { + logger.warn("Search idle is not supported for indices with replicas using 'replication.type: SEGMENT'"); + } this.searchIdleAfter = searchIdleAfter; } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 8541a1f5e554b..5deaf5f9a483a 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4282,12 +4282,9 @@ public boolean scheduledRefresh() { boolean listenerNeedsRefresh = refreshListeners.refreshNeeded(); if (isReadAllowed() && (listenerNeedsRefresh || getEngine().refreshNeeded())) { if (listenerNeedsRefresh == false // if we have a listener that is waiting for a refresh we need to force it + && isSearchIdleSupported() && isSearchIdle() && indexSettings.isExplicitRefresh() == false - && indexSettings.isSegRepEnabled() == false - // Indices with segrep enabled will never wait on a refresh and ignore shard idle. Primary shards push out new segments only - // after a refresh, so we don't want to wait for a search to trigger that cycle. Replicas will only refresh after receiving - // a new set of segments. && active.get()) { // it must be active otherwise we might not free up segment memory once the shard became inactive // lets skip this refresh since we are search idle and // don't necessarily need to refresh. the next searcher access will register a refreshListener and that will @@ -4315,6 +4312,19 @@ public final boolean isSearchIdle() { return (threadPool.relativeTimeInMillis() - lastSearcherAccess.get()) >= indexSettings.getSearchIdleAfter().getMillis(); } + /** + * + * Returns true if this shard supports search idle. + * + * Indices using Segment Replication will ignore search idle unless there are no replicas. + * Primary shards push out new segments only + * after a refresh, so we don't want to wait for a search to trigger that cycle. Replicas will only refresh after receiving + * a new set of segments. + */ + public final boolean isSearchIdleSupported() { + return indexSettings.isSegRepEnabled() == false || indexSettings.getNumberOfReplicas() == 0; + } + /** * Returns the last timestamp the searcher was accessed. This is a relative timestamp in milliseconds. */ diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 1f5980ba9bfe0..aa590358efa25 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -410,37 +410,68 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } public void testIgnoreShardIdle() throws Exception { - try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { + Settings updatedSettings = Settings.builder() + .put(settings) + .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO) + .build(); + try (ReplicationGroup shards = createGroup(1, updatedSettings, new NRTReplicationEngineFactory())) { shards.startAll(); final IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); - - final int numDocs = shards.indexDocs(randomInt(10)); - primary.refresh("test"); + final int numDocs = shards.indexDocs(randomIntBetween(1, 10)); + // ensure search idle conditions are met. + assertTrue(primary.isSearchIdle()); + assertTrue(replica.isSearchIdle()); + + // invoke scheduledRefresh, returns true if refresh is immediately invoked. + assertTrue(primary.scheduledRefresh()); + // replica would always return false here as there is no indexed doc to refresh on. + assertFalse(replica.scheduledRefresh()); + + // assert there is no pending refresh + assertFalse(primary.hasRefreshPending()); + assertFalse(replica.hasRefreshPending()); + shards.refresh("test"); replicateSegments(primary, shards.getReplicas()); shards.assertAllEqual(numDocs); + } + } - primary.scheduledRefresh(); - replica.scheduledRefresh(); - - primary.awaitShardSearchActive(b -> assertFalse("A new RefreshListener should not be registered", b)); - replica.awaitShardSearchActive(b -> assertFalse("A new RefreshListener should not be registered", b)); - - // Update the search_idle setting, this will put both shards into search idle. - Settings updatedSettings = Settings.builder() - .put(settings) - .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO) - .build(); - primary.indexSettings().getScopedSettings().applySettings(updatedSettings); - replica.indexSettings().getScopedSettings().applySettings(updatedSettings); - - primary.scheduledRefresh(); - replica.scheduledRefresh(); + public void testShardIdle_Docrep() throws Exception { + Settings settings = Settings.builder() + .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .build(); + try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { + shards.startAll(); + final IndexShard primary = shards.getPrimary(); + final IndexShard replica = shards.getReplicas().get(0); + final int numDocs = shards.indexDocs(randomIntBetween(1, 10)); + // ensure search idle conditions are met. + assertTrue(primary.isSearchIdle()); + assertTrue(replica.isSearchIdle()); + assertFalse(primary.scheduledRefresh()); + assertFalse(replica.scheduledRefresh()); + assertTrue(primary.hasRefreshPending()); + assertTrue(replica.hasRefreshPending()); + shards.refresh("test"); + shards.assertAllEqual(numDocs); + } + } - // Shards without segrep will register a new RefreshListener on the engine and return true when registered, - // assert with segrep enabled that awaitShardSearchActive does not register a listener. - primary.awaitShardSearchActive(b -> assertFalse("A new RefreshListener should not be registered", b)); - replica.awaitShardSearchActive(b -> assertFalse("A new RefreshListener should not be registered", b)); + public void testShardIdleWithNoReplicas() throws Exception { + Settings updatedSettings = Settings.builder() + .put(settings) + .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO) + .build(); + try (ReplicationGroup shards = createGroup(0, updatedSettings, new NRTReplicationEngineFactory())) { + shards.startAll(); + final IndexShard primary = shards.getPrimary(); + shards.indexDocs(randomIntBetween(1, 10)); + // ensure search idle conditions are met. + assertTrue(primary.isSearchIdle()); + assertFalse(primary.scheduledRefresh()); + assertTrue(primary.hasRefreshPending()); } } From 43d0ccd237a442ba4bf41f04733a6779d1735e12 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 17 Jul 2023 12:20:40 -0400 Subject: [PATCH 127/186] update grpc (#8726) Signed-off-by: Stephen Crawford --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- plugins/repository-gcs/licenses/grpc-context-1.46.0.jar.sha1 | 1 - plugins/repository-gcs/licenses/grpc-context-1.56.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/grpc-context-1.46.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/grpc-context-1.56.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d3dfe121a8522..4dafe5c55ed47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) +- Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 9bf1e8ac856b1..1e8e358833cc9 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -86,7 +86,7 @@ dependencies { api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" api "commons-codec:commons-codec:${versions.commonscodec}" api 'org.threeten:threetenbp:1.4.4' - api 'io.grpc:grpc-context:1.46.0' + api 'io.grpc:grpc-context:1.56.1' api 'io.opencensus:opencensus-api:0.31.1' api 'io.opencensus:opencensus-contrib-http-util:0.31.1' diff --git a/plugins/repository-gcs/licenses/grpc-context-1.46.0.jar.sha1 b/plugins/repository-gcs/licenses/grpc-context-1.46.0.jar.sha1 deleted file mode 100644 index e4d4c881492a2..0000000000000 --- a/plugins/repository-gcs/licenses/grpc-context-1.46.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -621d765a459758cdcd9aa4b4795392a4353e3576 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/grpc-context-1.56.1.jar.sha1 b/plugins/repository-gcs/licenses/grpc-context-1.56.1.jar.sha1 new file mode 100644 index 0000000000000..dbc1ea470dadd --- /dev/null +++ b/plugins/repository-gcs/licenses/grpc-context-1.56.1.jar.sha1 @@ -0,0 +1 @@ +3681b1caf41af1da0c4a3ffec47ab4a3d907c190 \ No newline at end of file From 688c53faf26460edac17e3380b1a0a7c2999a922 Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Mon, 17 Jul 2023 22:07:53 +0530 Subject: [PATCH 128/186] ClusterShardLimitIT.testCreateIndexWithMaxClusterShardSetting (#8708) Signed-off-by: Rishav Sagar Co-authored-by: Rishav Sagar --- .../java/org/opensearch/cluster/shards/ClusterShardLimitIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java index 5d6bed4629587..23335f6e82ef1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java @@ -246,6 +246,7 @@ public void testIndexCreationOverLimitForDotIndexesFail() { assertFalse(clusterState.getMetadata().hasIndex(".test-index")); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6287") public void testCreateIndexWithMaxClusterShardSetting() { int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); From 7642e435eecd9d5e390dc7906bff38ae2483c304 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 17 Jul 2023 12:45:25 -0700 Subject: [PATCH 129/186] Fix flaky test testDropPrimaryDuringReplication. (#8715) This change fixes testDropPrimaryDuringReplication and an edge case where after an updateSegments call on NRTReplicationReaderManager the reader is not actually refreshed because of another concurrent refresh call. Fixes by using a blocking refresh and removes unnecessary synchronization around the updatesegments method to avoid a deadlock case where the concurrent refresh picks up the new segments but is unable to acquire the object monitor to refresh internally in ReferenceManager.swapReference. Signed-off-by: Marc Handalian --- .../engine/NRTReplicationReaderManager.java | 4 +- .../NRTReplicationReaderManagerTests.java | 88 +++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java index 268ba1a436393..35409437f605a 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java @@ -98,12 +98,12 @@ protected OpenSearchDirectoryReader refreshIfNeeded(OpenSearchDirectoryReader re * @param infos {@link SegmentInfos} infos * @throws IOException - When Refresh fails with an IOException. */ - public synchronized void updateSegments(SegmentInfos infos) throws IOException { + public void updateSegments(SegmentInfos infos) throws IOException { // roll over the currentInfo's generation, this ensures the on-disk gen // is always increased. infos.updateGeneration(currentInfos); currentInfos = infos; - maybeRefresh(); + maybeRefreshBlocking(); } public SegmentInfos getSegmentInfos() { diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java new file mode 100644 index 0000000000000..98f1a416731e4 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.util.Version; +import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; +import org.opensearch.index.store.Store; + +import java.io.IOException; + +public class NRTReplicationReaderManagerTests extends EngineTestCase { + + public void testCreateNRTreaderManager() throws IOException { + try (final Store store = createStore()) { + store.createEmpty(Version.LATEST); + final DirectoryReader reader = DirectoryReader.open(store.directory()); + final SegmentInfos initialInfos = ((StandardDirectoryReader) reader).getSegmentInfos(); + NRTReplicationReaderManager readerManager = new NRTReplicationReaderManager( + OpenSearchDirectoryReader.wrap(reader, shardId), + (files) -> {}, + (files) -> {} + ); + assertEquals(initialInfos, readerManager.getSegmentInfos()); + try (final OpenSearchDirectoryReader acquire = readerManager.acquire()) { + assertNull(readerManager.refreshIfNeeded(acquire)); + } + + // create an updated infos + final SegmentInfos infos_2 = readerManager.getSegmentInfos().clone(); + infos_2.changed(); + + readerManager.updateSegments(infos_2); + assertEquals(infos_2, readerManager.getSegmentInfos()); + try (final OpenSearchDirectoryReader acquire = readerManager.acquire()) { + final StandardDirectoryReader standardReader = NRTReplicationReaderManager.unwrapStandardReader(acquire); + assertEquals(infos_2, standardReader.getSegmentInfos()); + } + } + } + + public void testUpdateSegmentsWhileRefreshing() throws IOException, InterruptedException { + try (final Store store = createStore()) { + store.createEmpty(Version.LATEST); + final DirectoryReader reader = DirectoryReader.open(store.directory()); + NRTReplicationReaderManager readerManager = new NRTReplicationReaderManager( + OpenSearchDirectoryReader.wrap(reader, shardId), + (files) -> {}, + (files) -> {} + ); + + final SegmentInfos infos_2 = readerManager.getSegmentInfos().clone(); + infos_2.changed(); + + Thread refreshThread = new Thread(() -> { + try { + readerManager.maybeRefresh(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + Thread updateThread = new Thread(() -> { + try { + readerManager.updateSegments(infos_2); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + refreshThread.start(); + updateThread.start(); + refreshThread.join(); + updateThread.join(); + try (final OpenSearchDirectoryReader acquire = readerManager.acquire()) { + final StandardDirectoryReader standardReader = NRTReplicationReaderManager.unwrapStandardReader(acquire); + assertEquals(infos_2.version, standardReader.getSegmentInfos().version); + } + assertEquals(infos_2, readerManager.getSegmentInfos()); + } + } +} From 8bd6b7cf988d08aeafc139f1ac0382cee7819889 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 17 Jul 2023 17:25:17 -0400 Subject: [PATCH 130/186] Manually update commons-codec from 1.15 to 1.16.0 (#8725) * update codec Signed-off-by: Stephen Crawford * Fix pr number Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Daniel (dB.) Doubrovkine Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dafe5c55ed47..d9c77e371ce69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) +- Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 018b63816c3f1..b1c1283e1dca5 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -102,7 +102,7 @@ dependencies { api localGroovy() - api 'commons-codec:commons-codec:1.15' + api 'commons-codec:commons-codec:1.16.0' api 'org.apache.commons:commons-compress:1.23.0' api 'org.apache.ant:ant:1.10.13' api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0' From 83c03dceb8c1f5e0cea43c71b35f087a2c7b4bb4 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 17 Jul 2023 18:21:36 -0400 Subject: [PATCH 131/186] Manually update Netflix Gradle info from 12.1.5 to 12.1.6 (#8724) * Update Netflix Gradle info Signed-off-by: Stephen Crawford * Add PR number Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Daniel (dB.) Doubrovkine Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c77e371ce69..d3eda3f852a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) +- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) ### Changed diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b1c1283e1dca5..4ca7cde1cebac 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -107,7 +107,7 @@ dependencies { api 'org.apache.ant:ant:1.10.13' api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0' api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0' - api 'com.netflix.nebula:gradle-info-plugin:12.1.5' + api 'com.netflix.nebula:gradle-info-plugin:12.1.6' api 'org.apache.rat:apache-rat:0.15' api 'commons-io:commons-io:2.13.0' api "net.java.dev.jna:jna:5.13.0" From 164fbb168316622831f71d82174790d2d10ec0cd Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Mon, 17 Jul 2023 22:52:28 -0500 Subject: [PATCH 132/186] [Refactor] Remaining HPPC to java.util collections (#8730) This commit refactors the remaining usages of hppc collections to java.util collections and completely removes the obsolete hppc dependency. Signed-off-by: Nicholas Walter Knize --- .../org/opensearch/common/util/BitMixer.java | 172 ++++++++ libs/core/.classpath1 | 6 - ...AbstractGeoBucketAggregationIntegTest.java | 16 +- .../aggregations/bucket/GeoHashGridIT.java | 11 +- ...ractGeoAggregatorModulePluginTestCase.java | 14 +- server/build.gradle | 1 - server/licenses/hppc-0.8.1.jar.sha1 | 1 - server/licenses/hppc-LICENSE.txt | 203 --------- server/licenses/hppc-NOTICE.txt | 11 - .../shards/IndicesShardStoreRequestIT.java | 33 +- .../cluster/ClusterStateDiffIT.java | 7 +- .../allocation/AwarenessAllocationIT.java | 31 +- .../cluster/routing/PrimaryAllocationIT.java | 11 +- .../index/store/CorruptedFileIT.java | 7 +- .../search/aggregations/CombiIT.java | 6 +- .../search/aggregations/EquivalenceIT.java | 5 +- .../aggregations/bucket/HistogramIT.java | 15 +- .../aggregations/bucket/MinDocCountIT.java | 4 +- .../search/scroll/DuelScrollIT.java | 5 +- .../DedicatedClusterSnapshotRestoreIT.java | 7 +- .../TransportClusterRerouteAction.java | 5 +- .../cluster/stats/ClusterStatsIndices.java | 10 +- .../cluster/stats/ClusterStatsNodes.java | 54 +-- .../mapping/put/PutMappingRequest.java | 4 +- .../shards/IndicesShardStoresResponse.java | 43 +- .../TransportIndicesShardStoresAction.java | 14 +- .../action/search/SearchPhaseController.java | 49 +-- .../action/support/ActiveShardCount.java | 6 +- .../TransportBroadcastReplicationAction.java | 8 +- .../action/termvectors/TermVectorsFields.java | 23 +- .../org/opensearch/cluster/ClusterInfo.java | 25 +- .../cluster/metadata/IndexMetadata.java | 6 +- .../metadata/MetadataIndexStateService.java | 18 +- .../cluster/routing/IndexRoutingTable.java | 42 +- .../cluster/routing/RoutingNodes.java | 28 +- .../cluster/routing/RoutingTable.java | 8 +- .../routing/WeightedRoutingService.java | 8 +- .../allocator/RemoteShardsBalancer.java | 14 +- .../decider/AwarenessAllocationDecider.java | 21 +- .../opensearch/common/FieldMemoryStats.java | 42 +- .../opensearch/common/collect/HppcMaps.java | 175 -------- .../common/collect/ImmutableOpenIntMap.java | 410 ------------------ .../lucene/search/MultiPhrasePrefixQuery.java | 9 +- .../search/function/RandomScoreFunction.java | 2 +- .../opensearch/common/recycler/Recyclers.java | 2 +- .../common/util/AbstractPagedHashMap.java | 1 - .../opensearch/common/util/BytesRefHash.java | 1 - .../opensearch/common/util/LongLongHash.java | 2 - .../common/util/ReorganizingLongHash.java | 1 - .../java/org/opensearch/gateway/Gateway.java | 9 +- .../http/AbstractHttpServerTransport.java | 7 +- .../index/engine/CombinedDeletionPolicy.java | 10 +- .../index/engine/CompletionStatsCache.java | 16 +- .../index/fielddata/ShardFieldData.java | 6 +- .../opensearch/index/mapper/ParseContext.java | 8 +- .../index/seqno/LocalCheckpointTracker.java | 25 +- .../index/seqno/ReplicationTracker.java | 6 +- .../opensearch/index/shard/IndexShard.java | 7 +- .../index/translog/MultiSnapshot.java | 5 +- .../index/translog/TranslogWriter.java | 12 +- .../indices/IndicesRequestCache.java | 7 +- .../rest/action/cat/RestAllocationAction.java | 7 +- .../rest/action/cat/RestFielddataAction.java | 7 +- .../opensearch/script/ScoreScriptUtils.java | 2 +- .../bucket/nested/NestedAggregator.java | 11 +- .../nested/ReverseNestedAggregator.java | 13 +- .../bucket/terms/IncludeExclude.java | 12 +- .../metrics/CardinalityAggregator.java | 2 +- .../metrics/TopHitsAggregator.java | 18 +- ...tilesBucketPipelineAggregationBuilder.java | 8 +- .../opensearch/search/dfs/AggregatedDfs.java | 29 +- .../org/opensearch/search/dfs/DfsPhase.java | 4 +- .../search/dfs/DfsSearchResult.java | 31 +- .../search/slice/DocValuesSliceQuery.java | 2 +- .../opensearch/snapshots/RestoreService.java | 6 +- .../org/opensearch/tasks/TaskManager.java | 15 +- .../opensearch/transport/TcpTransport.java | 6 +- .../IndicesShardStoreResponseTests.java | 7 +- .../action/support/ActiveShardCountTests.java | 17 +- .../health/ClusterStateHealthTests.java | 61 ++- .../cluster/routing/UnassignedInfoTests.java | 4 +- .../RemoteShardsAllocateUnassignedTests.java | 8 +- .../RemoteShardsRebalanceShardsTests.java | 18 +- .../allocation/ThrottlingAllocationTests.java | 3 +- .../decider/DiskThresholdDeciderTests.java | 3 +- ...storeInProgressAllocationDeciderTests.java | 5 +- .../common/FieldMemoryStatsTests.java | 43 +- .../opensearch/common/hppc/HppcMapsTests.java | 111 ----- .../common/util/BytesRefHashTests.java | 9 +- .../opensearch/common/util/LongHashTests.java | 12 +- .../common/util/LongObjectHashMapTests.java | 10 +- .../engine/CombinedDeletionPolicyTests.java | 3 +- .../seqno/LocalCheckpointTrackerTests.java | 6 +- .../index/translog/MultiSnapshotTests.java | 10 +- .../index/translog/SnapshotMatchers.java | 6 +- .../range/BinaryRangeAggregatorTests.java | 14 +- .../metrics/AbstractGeoTestCase.java | 15 +- .../HyperLogLogPlusPlusSparseTests.java | 2 +- .../metrics/HyperLogLogPlusPlusTests.java | 7 +- .../metrics/InternalCardinalityTests.java | 2 +- .../slice/DocValuesSliceQueryTests.java | 2 +- .../InternalSnapshotsInfoServiceTests.java | 4 +- .../coordination/LinearizabilityChecker.java | 19 +- .../opensearch/test/InternalTestCluster.java | 17 +- 104 files changed, 730 insertions(+), 1576 deletions(-) create mode 100644 libs/common/src/main/java/org/opensearch/common/util/BitMixer.java delete mode 100644 server/licenses/hppc-0.8.1.jar.sha1 delete mode 100644 server/licenses/hppc-LICENSE.txt delete mode 100644 server/licenses/hppc-NOTICE.txt delete mode 100644 server/src/main/java/org/opensearch/common/collect/HppcMaps.java delete mode 100644 server/src/main/java/org/opensearch/common/collect/ImmutableOpenIntMap.java delete mode 100644 server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java diff --git a/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java b/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java new file mode 100644 index 0000000000000..8762217916c7a --- /dev/null +++ b/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java @@ -0,0 +1,172 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * HPPC + * + * Copyright (C) 2010-2022 Carrot Search s.c. + * All rights reserved. + * + * Refer to the full license file "LICENSE.txt": + * https://github.com/carrotsearch/hppc/blob/master/LICENSE.txt + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.common.util; + +/** + * Bit mixing utilities from carrotsearch.hppc. + * + * Licensed under ALv2. This is pulled in directly to avoid a full hppc dependency. + * + * The purpose of these methods is to evenly distribute key space over int32 + * range. + */ +public final class BitMixer { + + // Don't bother mixing very small key domains much. + public static int mix(byte key) { + return key * PHI_C32; + } + + public static int mix(byte key, int seed) { + return (key ^ seed) * PHI_C32; + } + + public static int mix(short key) { + return mixPhi(key); + } + + public static int mix(short key, int seed) { + return mixPhi(key ^ seed); + } + + public static int mix(char key) { + return mixPhi(key); + } + + public static int mix(char key, int seed) { + return mixPhi(key ^ seed); + } + + // Better mix for larger key domains. + public static int mix(int key) { + return mix32(key); + } + + public static int mix(int key, int seed) { + return mix32(key ^ seed); + } + + public static int mix(float key) { + return mix32(Float.floatToIntBits(key)); + } + + public static int mix(float key, int seed) { + return mix32(Float.floatToIntBits(key) ^ seed); + } + + public static int mix(double key) { + return (int) mix64(Double.doubleToLongBits(key)); + } + + public static int mix(double key, int seed) { + return (int) mix64(Double.doubleToLongBits(key) ^ seed); + } + + public static int mix(long key) { + return (int) mix64(key); + } + + public static int mix(long key, int seed) { + return (int) mix64(key ^ seed); + } + + public static int mix(Object key) { + return key == null ? 0 : mix32(key.hashCode()); + } + + public static int mix(Object key, int seed) { + return key == null ? 0 : mix32(key.hashCode() ^ seed); + } + + /** + * MH3's plain finalization step. + */ + public static int mix32(int k) { + k = (k ^ (k >>> 16)) * 0x85ebca6b; + k = (k ^ (k >>> 13)) * 0xc2b2ae35; + return k ^ (k >>> 16); + } + + /** + * Computes David Stafford variant 9 of 64bit mix function (MH3 finalization step, + * with different shifts and constants). + * + * Variant 9 is picked because it contains two 32-bit shifts which could be possibly + * optimized into better machine code. + * + * @see "http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html" + */ + public static long mix64(long z) { + z = (z ^ (z >>> 32)) * 0x4cd6944c5cc20b6dL; + z = (z ^ (z >>> 29)) * 0xfc12c5b19d3259e9L; + return z ^ (z >>> 32); + } + + /* + * Golden ratio bit mixers. + */ + + private static final int PHI_C32 = 0x9e3779b9; + private static final long PHI_C64 = 0x9e3779b97f4a7c15L; + + public static int mixPhi(byte k) { + final int h = k * PHI_C32; + return h ^ (h >>> 16); + } + + public static int mixPhi(char k) { + final int h = k * PHI_C32; + return h ^ (h >>> 16); + } + + public static int mixPhi(short k) { + final int h = k * PHI_C32; + return h ^ (h >>> 16); + } + + public static int mixPhi(int k) { + final int h = k * PHI_C32; + return h ^ (h >>> 16); + } + + public static int mixPhi(float k) { + final int h = Float.floatToIntBits(k) * PHI_C32; + return h ^ (h >>> 16); + } + + public static int mixPhi(double k) { + final long h = Double.doubleToLongBits(k) * PHI_C64; + return (int) (h ^ (h >>> 32)); + } + + public static int mixPhi(long k) { + final long h = k * PHI_C64; + return (int) (h ^ (h >>> 32)); + } + + public static int mixPhi(Object k) { + final int h = (k == null ? 0 : k.hashCode() * PHI_C32); + return h ^ (h >>> 16); + } +} diff --git a/libs/core/.classpath1 b/libs/core/.classpath1 index 3a20aa700e9e9..3586b0ee921fb 100644 --- a/libs/core/.classpath1 +++ b/libs/core/.classpath1 @@ -253,12 +253,6 @@ - - - - - - diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java index 30088c1acb136..d9ff3e8f473ef 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java @@ -8,8 +8,6 @@ package org.opensearch.geo.search.aggregations.bucket; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.ObjectIntMap; import org.apache.lucene.geo.GeoEncodingUtils; import org.opensearch.Version; import org.opensearch.action.index.IndexRequestBuilder; @@ -26,8 +24,10 @@ import org.opensearch.test.VersionUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Set; @@ -51,11 +51,11 @@ public abstract class AbstractGeoBucketAggregationIntegTest extends GeoModulePlu protected static Rectangle boundingRectangleForGeoShapesAgg; - protected static ObjectIntMap expectedDocsCountForGeoShapes; + protected static Map expectedDocsCountForGeoShapes; - protected static ObjectIntMap expectedDocCountsForSingleGeoPoint; + protected static Map expectedDocCountsForSingleGeoPoint; - protected static ObjectIntMap multiValuedExpectedDocCountsGeoPoint; + protected static Map multiValuedExpectedDocCountsGeoPoint; protected static final String GEO_SHAPE_FIELD_NAME = "location_geo_shape"; @@ -82,7 +82,7 @@ protected boolean forbidPrivateIndexSettings() { * @throws Exception thrown during index creation. */ protected void prepareGeoShapeIndexForAggregations(final Random random) throws Exception { - expectedDocsCountForGeoShapes = new ObjectIntHashMap<>(); + expectedDocsCountForGeoShapes = new HashMap<>(); final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); final List geoshapes = new ArrayList<>(); assertAcked(prepareCreate(GEO_SHAPE_INDEX_NAME).setSettings(settings).setMapping(GEO_SHAPE_FIELD_NAME, "type" + "=geo_shape")); @@ -129,7 +129,7 @@ protected void prepareGeoShapeIndexForAggregations(final Random random) throws E * @throws Exception thrown during index creation. */ protected void prepareSingleValueGeoPointIndex(final Random random) throws Exception { - expectedDocCountsForSingleGeoPoint = new ObjectIntHashMap<>(); + expectedDocCountsForSingleGeoPoint = new HashMap<>(); createIndex("idx_unmapped"); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, version) @@ -155,7 +155,7 @@ protected void prepareSingleValueGeoPointIndex(final Random random) throws Excep } protected void prepareMultiValuedGeoPointIndex(final Random random) throws Exception { - multiValuedExpectedDocCountsGeoPoint = new ObjectIntHashMap<>(); + multiValuedExpectedDocCountsGeoPoint = new HashMap<>(); final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); final List cities = new ArrayList<>(); assertAcked( diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java index 3d4cd430a77e2..e3ca03aa495ab 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java @@ -31,8 +31,6 @@ package org.opensearch.geo.search.aggregations.bucket; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.cursors.ObjectIntCursor; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; @@ -49,6 +47,7 @@ import org.opensearch.search.aggregations.bucket.filter.Filter; import org.opensearch.test.OpenSearchIntegTestCase; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Random; @@ -70,7 +69,7 @@ public void setupSuiteScopeCluster() throws Exception { Random random = random(); // Creating a BB for limiting the number buckets generated during aggregation boundingRectangleForGeoShapesAgg = getGridAggregationBoundingBox(random); - expectedDocCountsForSingleGeoPoint = new ObjectIntHashMap<>(); + expectedDocCountsForSingleGeoPoint = new HashMap<>(); prepareSingleValueGeoPointIndex(random); prepareMultiValuedGeoPointIndex(random); prepareGeoShapeIndexForAggregations(random); @@ -232,9 +231,9 @@ public void testTopMatch() { String geohash = cell.getKeyAsString(); long bucketCount = cell.getDocCount(); int expectedBucketCount = 0; - for (ObjectIntCursor cursor : expectedDocCountsForSingleGeoPoint) { - if (cursor.key.length() == precision) { - expectedBucketCount = Math.max(expectedBucketCount, cursor.value); + for (var cursor : expectedDocCountsForSingleGeoPoint.entrySet()) { + if (cursor.getKey().length() == precision) { + expectedBucketCount = Math.max(expectedBucketCount, cursor.getValue()); } } assertNotSame(bucketCount, 0); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java index 03ed2ea6d1e3b..f7c9747e1a163 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java @@ -8,10 +8,6 @@ package org.opensearch.geo.search.aggregations.metrics; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.ObjectIntMap; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.ObjectObjectMap; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Strings; @@ -32,7 +28,9 @@ import org.opensearch.search.sort.SortOrder; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.IntStream; import static org.hamcrest.Matchers.equalTo; @@ -65,8 +63,8 @@ public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModul protected static Geometry[] geoShapesValues; protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, singleCentroid, multiCentroid, unmappedCentroid, geoShapeTopLeft, geoShapeBottomRight; - protected static ObjectIntMap expectedDocCountsForGeoHash = null; - protected static ObjectObjectMap expectedCentroidsForGeoHash = null; + protected static Map expectedDocCountsForGeoHash = null; + protected static Map expectedCentroidsForGeoHash = null; @Override public void setupSuiteScopeCluster() throws Exception { @@ -98,8 +96,8 @@ public void setupSuiteScopeCluster() throws Exception { numDocs = randomIntBetween(6, 20); numUniqueGeoPoints = randomIntBetween(1, numDocs); - expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); - expectedCentroidsForGeoHash = new ObjectObjectHashMap<>(numDocs * 2); + expectedDocCountsForGeoHash = new HashMap<>(numDocs * 2); + expectedCentroidsForGeoHash = new HashMap<>(numDocs * 2); singleValues = new GeoPoint[numUniqueGeoPoints]; for (int i = 0; i < singleValues.length; i++) { diff --git a/server/build.gradle b/server/build.gradle index e80aa0f77a164..3fde1b745c546 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -128,7 +128,6 @@ dependencies { // utilities api project(":libs:opensearch-cli") - api 'com.carrotsearch:hppc:0.8.1' // time handling, remove with java 8 time api "joda-time:joda-time:${versions.joda}" diff --git a/server/licenses/hppc-0.8.1.jar.sha1 b/server/licenses/hppc-0.8.1.jar.sha1 deleted file mode 100644 index 47684ed023210..0000000000000 --- a/server/licenses/hppc-0.8.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ffc7ba8f289428b9508ab484b8001dea944ae603 \ No newline at end of file diff --git a/server/licenses/hppc-LICENSE.txt b/server/licenses/hppc-LICENSE.txt deleted file mode 100644 index 31467575cdbfe..0000000000000 --- a/server/licenses/hppc-LICENSE.txt +++ /dev/null @@ -1,203 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2010-2013, Carrot Search s.c., Boznicza 11/56, Poznan, Poland - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/server/licenses/hppc-NOTICE.txt b/server/licenses/hppc-NOTICE.txt deleted file mode 100644 index 1d8842c0bc69d..0000000000000 --- a/server/licenses/hppc-NOTICE.txt +++ /dev/null @@ -1,11 +0,0 @@ -ACKNOWLEDGEMENT -=============== - -HPPC borrowed code, ideas or both from: - - * Apache Lucene, http://lucene.apache.org/ - (Apache license) - * Fastutil, http://fastutil.di.unimi.it/ - (Apache license) - * Koloboke, https://github.com/OpenHFT/Koloboke - (Apache license) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index 52b2ea93fe071..fe5ec9227e844 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -32,8 +32,6 @@ package org.opensearch.action.admin.indices.shards; -import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.index.CorruptIndexException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.client.Requests; @@ -42,7 +40,6 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; @@ -101,10 +98,10 @@ public void testBasic() throws Exception { // all shards response = client().admin().indices().shardStores(Requests.indicesShardStoresRequest(index).shardStatuses("all")).get(); assertThat(response.getStoreStatuses().containsKey(index), equalTo(true)); - ImmutableOpenIntMap> shardStores = response.getStoreStatuses().get(index); + final Map> shardStores = response.getStoreStatuses().get(index); assertThat(shardStores.values().size(), equalTo(2)); - for (ObjectCursor> shardStoreStatuses : shardStores.values()) { - for (IndicesShardStoresResponse.StoreStatus storeStatus : shardStoreStatuses.value) { + for (var shardStoreStatuses : shardStores.values()) { + for (IndicesShardStoresResponse.StoreStatus storeStatus : shardStoreStatuses) { assertThat(storeStatus.getAllocationId(), notNullValue()); assertThat(storeStatus.getNode(), notNullValue()); assertThat(storeStatus.getStoreException(), nullValue()); @@ -123,13 +120,13 @@ public void testBasic() throws Exception { List unassignedShards = clusterState.routingTable().index(index).shardsWithState(ShardRoutingState.UNASSIGNED); response = client().admin().indices().shardStores(Requests.indicesShardStoresRequest(index)).get(); assertThat(response.getStoreStatuses().containsKey(index), equalTo(true)); - ImmutableOpenIntMap> shardStoresStatuses = response.getStoreStatuses().get(index); + final Map> shardStoresStatuses = response.getStoreStatuses().get(index); assertThat(shardStoresStatuses.size(), equalTo(unassignedShards.size())); - for (IntObjectCursor> storesStatus : shardStoresStatuses) { - assertThat("must report for one store", storesStatus.value.size(), equalTo(1)); + for (var storesStatus : shardStoresStatuses.values()) { + assertThat("must report for one store", storesStatus.size(), equalTo(1)); assertThat( "reported store should be primary", - storesStatus.value.get(0).getAllocationStatus(), + storesStatus.get(0).getAllocationStatus(), equalTo(IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY) ); } @@ -150,7 +147,7 @@ public void testIndices() throws Exception { .indices() .shardStores(Requests.indicesShardStoresRequest().shardStatuses("all")) .get(); - Map>> shardStatuses = response.getStoreStatuses(); + Map>> shardStatuses = response.getStoreStatuses(); assertThat(shardStatuses.containsKey(index1), equalTo(true)); assertThat(shardStatuses.containsKey(index2), equalTo(true)); assertThat(shardStatuses.get(index1).size(), equalTo(2)); @@ -205,21 +202,21 @@ public void testCorruptedShards() throws Exception { assertBusy(() -> { // IndicesClusterStateService#failAndRemoveShard() called asynchronously but we need it to have completed here. IndicesShardStoresResponse rsp = client().admin().indices().prepareShardStores(index).setShardStatuses("all").get(); - ImmutableOpenIntMap> shardStatuses = rsp.getStoreStatuses().get(index); + final Map> shardStatuses = rsp.getStoreStatuses().get(index); assertNotNull(shardStatuses); assertThat(shardStatuses.size(), greaterThan(0)); - for (IntObjectCursor> shardStatus : shardStatuses) { - for (IndicesShardStoresResponse.StoreStatus status : shardStatus.value) { - if (corruptedShardIDMap.containsKey(shardStatus.key) - && corruptedShardIDMap.get(shardStatus.key).contains(status.getNode().getName())) { + for (var shardStatus : shardStatuses.entrySet()) { + for (IndicesShardStoresResponse.StoreStatus status : shardStatus.getValue()) { + if (corruptedShardIDMap.containsKey(shardStatus.getKey()) + && corruptedShardIDMap.get(shardStatus.getKey()).contains(status.getNode().getName())) { assertThat( - "shard [" + shardStatus.key + "] is failed on node [" + status.getNode().getName() + "]", + "shard [" + shardStatus.getKey() + "] is failed on node [" + status.getNode().getName() + "]", status.getStoreException(), notNullValue() ); } else { assertNull( - "shard [" + shardStatus.key + "] is not failed on node [" + status.getNode().getName() + "]", + "shard [" + shardStatus.getKey() + "] is not failed on node [" + status.getNode().getName() + "]", status.getStoreException() ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java index e5a46c3c6bb9f..147be425f93b3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java @@ -32,7 +32,6 @@ package org.opensearch.cluster; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.cluster.block.ClusterBlock; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; @@ -347,16 +346,16 @@ private IndexRoutingTable randomIndexRoutingTable(String index, String[] nodeIds */ private IndexRoutingTable randomChangeToIndexRoutingTable(IndexRoutingTable original, String[] nodes) { IndexRoutingTable.Builder builder = IndexRoutingTable.builder(original.getIndex()); - for (ObjectCursor indexShardRoutingTable : original.shards().values()) { + for (var indexShardRoutingTable : original.shards().values()) { Set availableNodes = Sets.newHashSet(nodes); - for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) { + for (ShardRouting shardRouting : indexShardRoutingTable.shards()) { availableNodes.remove(shardRouting.currentNodeId()); if (shardRouting.relocating()) { availableNodes.remove(shardRouting.relocatingNodeId()); } } - for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) { + for (ShardRouting shardRouting : indexShardRoutingTable.shards()) { final ShardRouting updatedShardRouting = randomChange(shardRouting, availableNodes); availableNodes.remove(updatedShardRouting.currentNodeId()); if (shardRouting.relocating()) { diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java index a8d4d016ce288..c69718d982f8b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.allocation; -import com.carrotsearch.hppc.ObjectIntHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; @@ -53,7 +52,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -126,11 +127,11 @@ public void testSimpleAwareness() throws Exception { assertThat("Some indices not closed", notClosedIndices, empty()); // verify that we have all the primaries on node3 - ObjectIntHashMap counts = new ObjectIntHashMap<>(); + final Map counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } @@ -182,12 +183,12 @@ public void testAwarenessZones() { assertThat(health.isTimedOut(), equalTo(false)); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - ObjectIntHashMap counts = new ObjectIntHashMap<>(); + final Map counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } @@ -232,12 +233,12 @@ public void testAwarenessZonesIncrementalNodes() { .actionGet(); assertThat(health.isTimedOut(), equalTo(false)); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - ObjectIntHashMap counts = new ObjectIntHashMap<>(); + Map counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } @@ -272,12 +273,12 @@ public void testAwarenessZonesIncrementalNodes() { assertThat(health.isTimedOut(), equalTo(false)); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - counts = new ObjectIntHashMap<>(); + counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } @@ -312,12 +313,12 @@ public void testAwarenessZonesIncrementalNodes() { assertThat(health.isTimedOut(), equalTo(false)); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - counts = new ObjectIntHashMap<>(); + counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } @@ -347,12 +348,12 @@ public void testAwarenessZonesIncrementalNodes() { assertThat(health.isTimedOut(), equalTo(false)); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - counts = new ObjectIntHashMap<>(); + counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } @@ -415,12 +416,12 @@ public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws E assertFalse(health.isTimedOut()); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - ObjectIntHashMap counts = new ObjectIntHashMap<>(); + final Map counts = new HashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { - counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1); + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java index 017f6fc29ef87..637cc96bdfc44 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/PrimaryAllocationIT.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.routing; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; import org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse; @@ -48,7 +47,6 @@ import org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentType; @@ -74,6 +72,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -316,16 +315,16 @@ public void testForceStaleReplicaToBePromotedToPrimary() throws Exception { boolean useStaleReplica = randomBoolean(); // if true, use stale replica, otherwise a completely empty copy logger.info("--> explicitly promote old primary shard"); final String idxName = "test"; - ImmutableOpenIntMap> storeStatuses = client().admin() + final Map> storeStatuses = client().admin() .indices() .prepareShardStores(idxName) .get() .getStoreStatuses() .get(idxName); ClusterRerouteRequestBuilder rerouteBuilder = client().admin().cluster().prepareReroute(); - for (IntObjectCursor> shardStoreStatuses : storeStatuses) { - int shardId = shardStoreStatuses.key; - IndicesShardStoresResponse.StoreStatus storeStatus = randomFrom(shardStoreStatuses.value); + for (var shardStoreStatuses : storeStatuses.entrySet()) { + int shardId = shardStoreStatuses.getKey(); + IndicesShardStoresResponse.StoreStatus storeStatus = randomFrom(shardStoreStatuses.getValue()); logger.info("--> adding allocation command for shard {}", shardId); // force allocation based on node id if (useStaleReplica) { diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java index 28753e495881f..d51e4bbff11b5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java @@ -31,7 +31,6 @@ package org.opensearch.index.store; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.SegmentCommitInfo; @@ -674,9 +673,9 @@ public void testReplicaCorruption() throws Exception { final IndicesShardStoresResponse stores = client().admin().indices().prepareShardStores(index.getName()).get(); - for (IntObjectCursor> shards : stores.getStoreStatuses().get(index.getName())) { - for (IndicesShardStoresResponse.StoreStatus store : shards.value) { - final ShardId shardId = new ShardId(index, shards.key); + for (var shards : stores.getStoreStatuses().get(index.getName()).entrySet()) { + for (IndicesShardStoresResponse.StoreStatus store : shards.getValue()) { + final ShardId shardId = new ShardId(index, shards.getKey()); if (store.getAllocationStatus().equals(IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED)) { for (Path path : findFilesToCorruptOnNode(store.getNode().getName(), shardId)) { try (OutputStream os = Files.newOutputStream(path)) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java index c5794c76e21de..d35a560b0986c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java @@ -32,8 +32,6 @@ package org.opensearch.search.aggregations; -import com.carrotsearch.hppc.IntIntHashMap; -import com.carrotsearch.hppc.IntIntMap; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -43,7 +41,9 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.hamcrest.Matchers; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -67,7 +67,7 @@ public void testMultipleAggsOnSameField_WithDifferentRequiredValueSourceType() t createIndex("idx"); IndexRequestBuilder[] builders = new IndexRequestBuilder[randomInt(30)]; - IntIntMap values = new IntIntHashMap(); + final Map values = new HashMap<>(); long missingValues = 0; for (int i = 0; i < builders.length; i++) { String name = "name_" + randomIntBetween(1, 10); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java index ccd16ff9238bb..21f833d5430db 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java @@ -32,7 +32,6 @@ package org.opensearch.search.aggregations; -import com.carrotsearch.hppc.IntHashSet; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; @@ -61,9 +60,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; @@ -230,7 +231,7 @@ public void testDuelTerms() throws Exception { final int numDocs = scaledRandomIntBetween(1000, 2000); final int maxNumTerms = randomIntBetween(10, 5000); - final IntHashSet valuesSet = new IntHashSet(); + final Set valuesSet = new HashSet<>(); cluster().wipeIndices("idx"); prepareCreate("idx").setMapping( jsonBuilder().startObject() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java index dae788abe0d10..dd495701c3ddb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java @@ -31,7 +31,6 @@ package org.opensearch.search.aggregations.bucket; -import com.carrotsearch.hppc.LongHashSet; import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; @@ -61,8 +60,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static java.util.Collections.emptyMap; @@ -396,7 +397,7 @@ public void testSingleValuedFieldOrderedByCountAsc() throws Exception { assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); - LongHashSet buckets = new LongHashSet(); + final Set buckets = new HashSet<>(); List histoBuckets = new ArrayList<>(histo.getBuckets()); long previousCount = Long.MIN_VALUE; for (int i = 0; i < numValueBuckets; ++i) { @@ -423,7 +424,7 @@ public void testSingleValuedFieldOrderedByCountDesc() throws Exception { assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); - LongHashSet buckets = new LongHashSet(); + final Set buckets = new HashSet<>(); List histoBuckets = new ArrayList<>(histo.getBuckets()); long previousCount = Long.MAX_VALUE; for (int i = 0; i < numValueBuckets; ++i) { @@ -497,7 +498,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAsc() throws Exception { assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); - LongHashSet visited = new LongHashSet(); + final Set visited = new HashSet<>(); double previousSum = Double.NEGATIVE_INFINITY; List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { @@ -539,7 +540,7 @@ public void testSingleValuedFieldOrderedBySubAggregationDesc() throws Exception assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); - LongHashSet visited = new LongHashSet(); + final Set visited = new HashSet<>(); double previousSum = Double.POSITIVE_INFINITY; List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { @@ -581,7 +582,7 @@ public void testSingleValuedFieldOrderedByMultiValuedSubAggregationDesc() throws assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); - LongHashSet visited = new LongHashSet(); + final Set visited = new HashSet<>(); double previousSum = Double.POSITIVE_INFINITY; List buckets = new ArrayList<>(histo.getBuckets()); @@ -625,7 +626,7 @@ public void testSingleValuedFieldOrderedBySubAggregationDescDeepOrderPath() thro assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); - LongHashSet visited = new LongHashSet(); + final Set visited = new HashSet<>(); double prevMax = asc ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java index 08e696245209e..48e2a3d8fa9e1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java @@ -32,8 +32,6 @@ package org.opensearch.search.aggregations.bucket; -import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongSet; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequest; @@ -122,7 +120,7 @@ public void setupSuiteScopeCluster() throws Exception { cardinality = randomIntBetween(8, 30); final List indexRequests = new ArrayList<>(); final Set stringTerms = new HashSet<>(); - final LongSet longTerms = new LongHashSet(); + final Set longTerms = new HashSet(); for (int i = 0; i < cardinality; ++i) { String stringTerm; do { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java index 19cf1ee3a0ee7..e0a54e9b4fc36 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java @@ -32,7 +32,6 @@ package org.opensearch.search.scroll; -import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.opensearch.action.index.IndexRequestBuilder; @@ -49,6 +48,8 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -145,7 +146,7 @@ private TestContext create(SearchType... searchTypes) throws Exception { boolean unevenRouting = randomBoolean(); int numMissingDocs = scaledRandomIntBetween(0, numDocs / 100); - IntHashSet missingDocs = new IntHashSet(numMissingDocs); + final Set missingDocs = new HashSet<>(numMissingDocs); for (int i = 0; i < numMissingDocs; i++) { while (!missingDocs.add(randomInt(numDocs))) { } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 0d10124b585f5..85c1a7251d143 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -32,9 +32,6 @@ package org.opensearch.snapshots; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; - import org.opensearch.Version; import org.opensearch.action.ActionFuture; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -113,8 +110,10 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -736,7 +735,7 @@ public boolean clearData(String nodeName) { ensureGreen("test-idx"); - IntSet reusedShards = new IntHashSet(); + final Set reusedShards = new HashSet<>(); List recoveryStates = client().admin() .indices() .prepareRecoveries("test-idx") diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index 0a4b98f9d4bb5..61f2a6301a2dd 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -58,7 +58,6 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.inject.Inject; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.Strings; @@ -152,11 +151,11 @@ private void verifyThenSubmitUpdate( IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(stalePrimaryAllocations.keySet().toArray(Strings.EMPTY_ARRAY)), new ActionListenerResponseHandler<>(ActionListener.wrap(response -> { - final Map>> status = response.getStoreStatuses(); + final Map>> status = response.getStoreStatuses(); Exception e = null; for (Map.Entry> entry : stalePrimaryAllocations.entrySet()) { final String index = entry.getKey(); - final ImmutableOpenIntMap> indexStatus = status.get(index); + final Map> indexStatus = status.get(index); if (indexStatus == null) { // The index in the stale primary allocation request was green and hence filtered out by the store status // request. We ignore it here since the relevant exception will be thrown by the reroute action later on. diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 02a516fffbfd3..63ac76ae65783 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -32,8 +32,6 @@ package org.opensearch.action.admin.cluster.stats; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -45,7 +43,9 @@ import org.opensearch.search.suggest.completion.CompletionStats; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Cluster Stats per index @@ -66,7 +66,7 @@ public class ClusterStatsIndices implements ToXContentFragment { private MappingStats mappings; public ClusterStatsIndices(List nodeResponses, MappingStats mappingStats, AnalysisStats analysisStats) { - ObjectObjectHashMap countsPerIndex = new ObjectObjectHashMap<>(); + Map countsPerIndex = new HashMap<>(); this.docs = new DocsStats(); this.store = new StoreStats(); @@ -101,8 +101,8 @@ public ClusterStatsIndices(List nodeResponses, Mapping shards = new ShardStats(); indexCount = countsPerIndex.size(); - for (ObjectObjectCursor indexCountsCursor : countsPerIndex) { - shards.addIndexShardCount(indexCountsCursor.value); + for (final ShardStats indexCountsCursor : countsPerIndex.values()) { + shards.addIndexShardCount(indexCountsCursor); } this.mappings = mappingStats; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 25a915833c7e2..699884ca0eab3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -32,8 +32,6 @@ package org.opensearch.action.admin.cluster.stats; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.cursors.ObjectIntCursor; import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; @@ -289,16 +287,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static class OsStats implements ToXContentFragment { final int availableProcessors; final int allocatedProcessors; - final ObjectIntHashMap names; - final ObjectIntHashMap prettyNames; + final Map names; + final Map prettyNames; final org.opensearch.monitor.os.OsStats.Mem mem; /** * Build the stats from information about each node. */ private OsStats(List nodeInfos, List nodeStatsList) { - this.names = new ObjectIntHashMap<>(); - this.prettyNames = new ObjectIntHashMap<>(); + final Map names = new HashMap<>(nodeInfos.size()); + final Map prettyNames = new HashMap<>(nodeInfos.size()); int availableProcessors = 0; int allocatedProcessors = 0; for (NodeInfo nodeInfo : nodeInfos) { @@ -306,12 +304,14 @@ private OsStats(List nodeInfos, List nodeStatsList) { allocatedProcessors += nodeInfo.getInfo(OsInfo.class).getAllocatedProcessors(); if (nodeInfo.getInfo(OsInfo.class).getName() != null) { - names.addTo(nodeInfo.getInfo(OsInfo.class).getName(), 1); + names.merge(nodeInfo.getInfo(OsInfo.class).getName(), 1, Integer::sum); } if (nodeInfo.getInfo(OsInfo.class).getPrettyName() != null) { - prettyNames.addTo(nodeInfo.getInfo(OsInfo.class).getPrettyName(), 1); + prettyNames.merge(nodeInfo.getInfo(OsInfo.class).getPrettyName(), 1, Integer::sum); } } + this.names = Collections.unmodifiableMap(names); + this.prettyNames = Collections.unmodifiableMap(prettyNames); this.availableProcessors = availableProcessors; this.allocatedProcessors = allocatedProcessors; @@ -365,11 +365,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.ALLOCATED_PROCESSORS, allocatedProcessors); builder.startArray(Fields.NAMES); { - for (ObjectIntCursor name : names) { + for (final Map.Entry name : names.entrySet()) { builder.startObject(); { - builder.field(Fields.NAME, name.key); - builder.field(Fields.COUNT, name.value); + builder.field(Fields.NAME, name.getKey()); + builder.field(Fields.COUNT, name.getValue()); } builder.endObject(); } @@ -377,11 +377,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endArray(); builder.startArray(Fields.PRETTY_NAMES); { - for (final ObjectIntCursor prettyName : prettyNames) { + for (final Map.Entry prettyName : prettyNames.entrySet()) { builder.startObject(); { - builder.field(Fields.PRETTY_NAME, prettyName.key); - builder.field(Fields.COUNT, prettyName.value); + builder.field(Fields.PRETTY_NAME, prettyName.getKey()); + builder.field(Fields.COUNT, prettyName.getValue()); } builder.endObject(); } @@ -502,7 +502,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws */ public static class JvmStats implements ToXContentFragment { - private final ObjectIntHashMap versions; + private final Map versions; private final long threads; private final long maxUptime; private final long heapUsed; @@ -512,15 +512,15 @@ public static class JvmStats implements ToXContentFragment { * Build from lists of information about each node. */ private JvmStats(List nodeInfos, List nodeStatsList) { - this.versions = new ObjectIntHashMap<>(); + final Map versions = new HashMap<>(nodeInfos.size()); long threads = 0; long maxUptime = 0; long heapMax = 0; long heapUsed = 0; for (NodeInfo nodeInfo : nodeInfos) { - versions.addTo(new JvmVersion(nodeInfo.getInfo(JvmInfo.class)), 1); + versions.merge(new JvmVersion(nodeInfo.getInfo(JvmInfo.class)), 1, Integer::sum); } - + this.versions = Collections.unmodifiableMap(versions); for (NodeStats nodeStats : nodeStatsList) { org.opensearch.monitor.jvm.JvmStats js = nodeStats.getJvm(); if (js == null) { @@ -541,7 +541,7 @@ private JvmStats(List nodeInfos, List nodeStatsList) { this.heapMax = heapMax; } - public ObjectIntHashMap getVersions() { + public Map getVersions() { return versions; } @@ -601,15 +601,15 @@ static final class Fields { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.humanReadableField(Fields.MAX_UPTIME_IN_MILLIS, Fields.MAX_UPTIME, new TimeValue(maxUptime)); builder.startArray(Fields.VERSIONS); - for (ObjectIntCursor v : versions) { + for (final Map.Entry v : versions.entrySet()) { builder.startObject(); - builder.field(Fields.VERSION, v.key.version); - builder.field(Fields.VM_NAME, v.key.vmName); - builder.field(Fields.VM_VERSION, v.key.vmVersion); - builder.field(Fields.VM_VENDOR, v.key.vmVendor); - builder.field(Fields.BUNDLED_JDK, v.key.bundledJdk); - builder.field(Fields.USING_BUNDLED_JDK, v.key.usingBundledJdk); - builder.field(Fields.COUNT, v.value); + builder.field(Fields.VERSION, v.getKey().version); + builder.field(Fields.VM_NAME, v.getKey().vmName); + builder.field(Fields.VM_VERSION, v.getKey().vmVersion); + builder.field(Fields.VM_VENDOR, v.getKey().vmVendor); + builder.field(Fields.BUNDLED_JDK, v.getKey().bundledJdk); + builder.field(Fields.USING_BUNDLED_JDK, v.getKey().usingBundledJdk); + builder.field(Fields.COUNT, v.getValue()); builder.endObject(); } builder.endArray(); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java index 19e6664956354..465a44556c081 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -32,7 +32,6 @@ package org.opensearch.action.admin.indices.mapping.put; -import com.carrotsearch.hppc.ObjectHashSet; import org.opensearch.OpenSearchGenerationException; import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; @@ -61,6 +60,7 @@ import java.util.Arrays; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.opensearch.action.ValidateActions.addValidationError; @@ -79,7 +79,7 @@ */ public class PutMappingRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable, ToXContentObject { - private static ObjectHashSet RESERVED_FIELDS = ObjectHashSet.from( + private static final Set RESERVED_FIELDS = Set.of( "_uid", "_id", "_type", diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java index 37d1ab07605b2..fed9ee01b385d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java @@ -32,12 +32,10 @@ package org.opensearch.action.admin.indices.shards; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionResponse; -import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.collect.ImmutableOpenIntMap; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -270,10 +268,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t } } - private final Map>> storeStatuses; + private final Map>> storeStatuses; private final List failures; - public IndicesShardStoresResponse(final Map>> storeStatuses, List failures) { + public IndicesShardStoresResponse(final Map>> storeStatuses, List failures) { this.storeStatuses = Collections.unmodifiableMap(storeStatuses); this.failures = failures; } @@ -284,14 +282,11 @@ public IndicesShardStoresResponse(final Map { - int indexEntries = i.readVInt(); - ImmutableOpenIntMap.Builder> shardEntries = ImmutableOpenIntMap.builder(); - for (int shardCount = 0; shardCount < indexEntries; shardCount++) { - shardEntries.put(i.readInt(), i.readList(StoreStatus::new)); - } - return shardEntries.build(); - }); + final Map>> storeStatuses = in.readMap( + StreamInput::readString, + i -> i.readMap(StreamInput::readInt, j -> j.readList(StoreStatus::new)) + ); + this.storeStatuses = Collections.unmodifiableMap(storeStatuses); failures = Collections.unmodifiableList(in.readList(Failure::readFailure)); } @@ -299,7 +294,7 @@ public IndicesShardStoresResponse(StreamInput in) throws IOException { * Returns {@link StoreStatus}s * grouped by their index names and shard ids. */ - public Map>> getStoreStatuses() { + public Map>> getStoreStatuses() { return storeStatuses; } @@ -313,13 +308,11 @@ public List getFailures() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeMap(storeStatuses, StreamOutput::writeString, (o, v) -> { - o.writeVInt(v.size()); - for (IntObjectCursor> shardStatusesEntry : v) { - o.writeInt(shardStatusesEntry.key); - o.writeCollection(shardStatusesEntry.value); - } - }); + out.writeMap( + storeStatuses, + StreamOutput::writeString, + (o, v) -> o.writeMap(v, StreamOutput::writeInt, StreamOutput::writeCollection) + ); out.writeList(failures); } @@ -334,14 +327,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.startObject(Fields.INDICES); - for (final Map.Entry>> indexShards : storeStatuses.entrySet()) { + for (final Map.Entry>> indexShards : storeStatuses.entrySet()) { builder.startObject(indexShards.getKey()); builder.startObject(Fields.SHARDS); - for (IntObjectCursor> shardStatusesEntry : indexShards.getValue()) { - builder.startObject(String.valueOf(shardStatusesEntry.key)); + for (final Map.Entry> shardStatusesEntry : indexShards.getValue().entrySet()) { + builder.startObject(String.valueOf(shardStatusesEntry.getKey())); builder.startArray(Fields.STORES); - for (StoreStatus storeStatus : shardStatusesEntry.value) { + for (StoreStatus storeStatus : shardStatusesEntry.getValue()) { builder.startObject(); storeStatus.toXContent(builder, params); builder.endObject(); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index cbb5c3d686cba..c1e6a64efbf2f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -53,7 +53,6 @@ import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; import org.opensearch.core.common.io.stream.StreamInput; @@ -231,18 +230,17 @@ protected synchronized void processAsyncFetch( } void finish() { - final Map>> indicesStoreStatusesBuilder = - new HashMap<>(); + final Map>> indicesStoreStatusesBuilder = new HashMap<>(); java.util.List failureBuilder = new ArrayList<>(); for (Response fetchResponse : fetchResponses) { - ImmutableOpenIntMap> indexStoreStatuses = + final Map> indexStoreStatuses = indicesStoreStatusesBuilder.get(fetchResponse.shardId.getIndexName()); - final ImmutableOpenIntMap.Builder> indexShardsBuilder; + final Map> indexShardsBuilder; if (indexStoreStatuses == null) { - indexShardsBuilder = ImmutableOpenIntMap.builder(); + indexShardsBuilder = new HashMap<>(); } else { - indexShardsBuilder = ImmutableOpenIntMap.builder(indexStoreStatuses); + indexShardsBuilder = new HashMap<>(indexStoreStatuses); } java.util.List storeStatuses = indexShardsBuilder.get( fetchResponse.shardId.id() @@ -269,7 +267,7 @@ void finish() { } CollectionUtil.timSort(storeStatuses); indexShardsBuilder.put(fetchResponse.shardId.id(), storeStatuses); - indicesStoreStatusesBuilder.put(fetchResponse.shardId.getIndexName(), indexShardsBuilder.build()); + indicesStoreStatusesBuilder.put(fetchResponse.shardId.getIndexName(), Collections.unmodifiableMap(indexShardsBuilder)); for (FailedNodeException failure : fetchResponse.failures) { failureBuilder.add( new IndicesShardStoresResponse.Failure( diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index c1df2fa913885..512d3295c4cfc 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -32,8 +32,6 @@ package org.opensearch.action.search; -import com.carrotsearch.hppc.ObjectObjectHashMap; - import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.FieldDoc; @@ -48,7 +46,6 @@ import org.apache.lucene.search.TotalHits.Relation; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.collect.HppcMaps; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.search.DocValueFormat; @@ -105,8 +102,8 @@ public SearchPhaseController( } public AggregatedDfs aggregateDfs(Collection results) { - ObjectObjectHashMap termStatistics = HppcMaps.newNoNullKeysMap(); - ObjectObjectHashMap fieldStatistics = HppcMaps.newNoNullKeysMap(); + final Map termStatistics = new HashMap<>(); + final Map fieldStatistics = new HashMap<>(); long aggMaxDoc = 0; for (DfsSearchResult lEntry : results) { final Term[] terms = lEntry.terms(); @@ -135,29 +132,25 @@ public AggregatedDfs aggregateDfs(Collection results) { } assert !lEntry.fieldStatistics().containsKey(null); - final Object[] keys = lEntry.fieldStatistics().keys; - final Object[] values = lEntry.fieldStatistics().values; - for (int i = 0; i < keys.length; i++) { - if (keys[i] != null) { - String key = (String) keys[i]; - CollectionStatistics value = (CollectionStatistics) values[i]; - if (value == null) { - continue; - } - assert key != null; - CollectionStatistics existing = fieldStatistics.get(key); - if (existing != null) { - CollectionStatistics merged = new CollectionStatistics( - key, - existing.maxDoc() + value.maxDoc(), - existing.docCount() + value.docCount(), - existing.sumTotalTermFreq() + value.sumTotalTermFreq(), - existing.sumDocFreq() + value.sumDocFreq() - ); - fieldStatistics.put(key, merged); - } else { - fieldStatistics.put(key, value); - } + for (var entry : lEntry.fieldStatistics().entrySet()) { + String key = entry.getKey(); + CollectionStatistics value = entry.getValue(); + if (value == null) { + continue; + } + assert key != null; + CollectionStatistics existing = fieldStatistics.get(key); + if (existing != null) { + CollectionStatistics merged = new CollectionStatistics( + key, + existing.maxDoc() + value.maxDoc(), + existing.docCount() + value.docCount(), + existing.sumTotalTermFreq() + value.sumTotalTermFreq(), + existing.sumDocFreq() + value.sumDocFreq() + ); + fieldStatistics.put(key, merged); + } else { + fieldStatistics.put(key, value); } } aggMaxDoc += lEntry.maxDoc(); diff --git a/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java b/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java index 3a687f946f08b..15275ba48fc6e 100644 --- a/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java +++ b/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java @@ -32,8 +32,6 @@ package org.opensearch.action.support; -import com.carrotsearch.hppc.cursors.IntObjectCursor; - import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -186,8 +184,8 @@ public boolean enoughShardsActive(final ClusterState clusterState, final String. if (waitForActiveShards == ActiveShardCount.DEFAULT) { waitForActiveShards = SETTING_WAIT_FOR_ACTIVE_SHARDS.get(indexMetadata.getSettings()); } - for (final IntObjectCursor shardRouting : indexRoutingTable.getShards()) { - if (waitForActiveShards.enoughShardsActive(shardRouting.value) == false) { + for (final IndexShardRoutingTable shardRouting : indexRoutingTable.getShards().values()) { + if (waitForActiveShards.enoughShardsActive(shardRouting) == false) { // not enough active shard copies yet return false; } diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java index 41f05ade7fdf0..116b46469df21 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -32,7 +32,6 @@ package org.opensearch.action.support.replication; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; @@ -154,11 +153,12 @@ protected List shards(Request request, ClusterState clusterState) { for (String index : concreteIndices) { IndexMetadata indexMetadata = clusterState.metadata().getIndices().get(index); if (indexMetadata != null) { - for (IntObjectCursor shardRouting : clusterState.getRoutingTable() + for (IndexShardRoutingTable shardRouting : clusterState.getRoutingTable() .indicesRouting() .get(index) - .getShards()) { - shardIds.add(shardRouting.value.shardId()); + .getShards() + .values()) { + shardIds.add(shardRouting.shardId()); } } } diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java index ff7a8de3a97d3..ed3dca815090d 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFields.java @@ -32,8 +32,6 @@ package org.opensearch.action.termvectors; -import com.carrotsearch.hppc.ObjectLongHashMap; -import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.apache.lucene.index.BaseTermsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.ImpactsEnum; @@ -51,7 +49,10 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import static org.apache.lucene.util.ArrayUtil.grow; @@ -135,7 +136,7 @@ */ public final class TermVectorsFields extends Fields { - private final ObjectLongHashMap fieldMap; + private final Map fieldMap; private final BytesReference termVectors; final boolean hasTermStatistic; final boolean hasFieldStatistic; @@ -157,10 +158,11 @@ public TermVectorsFields(BytesReference headerRef, BytesReference termVectors) t hasFieldStatistic = header.readBoolean(); hasScores = header.readBoolean(); final int numFields = header.readVInt(); - fieldMap = new ObjectLongHashMap<>(numFields); + final Map fieldMap = new HashMap<>(numFields); for (int i = 0; i < numFields; i++) { fieldMap.put((header.readString()), header.readVLong()); } + this.fieldMap = Collections.unmodifiableMap(fieldMap); } // reference to the term vector data this.termVectors = termVectors; @@ -168,8 +170,8 @@ public TermVectorsFields(BytesReference headerRef, BytesReference termVectors) t @Override public Iterator iterator() { - final Iterator> iterator = fieldMap.iterator(); - return new Iterator() { + final Iterator> iterator = fieldMap.entrySet().iterator(); + return new Iterator<>() { @Override public boolean hasNext() { return iterator.hasNext(); @@ -177,7 +179,7 @@ public boolean hasNext() { @Override public String next() { - return iterator.next().key; + return iterator.next().getKey(); } @Override @@ -191,12 +193,11 @@ public void remove() { public Terms terms(String field) throws IOException { // first, find where in the termVectors bytes the actual term vector for // this field is stored - final int keySlot = fieldMap.indexOf(field); - if (keySlot < 0) { + final Long keySlot = fieldMap.get(field); + if (keySlot == null) { return null; // we don't have it. } - long readOffset = fieldMap.indexGet(keySlot); - return new TermVector(termVectors, readOffset); + return new TermVector(termVectors, keySlot); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java index cac98a262df53..3793b5094a4cb 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java @@ -32,8 +32,6 @@ package org.opensearch.cluster; -import com.carrotsearch.hppc.ObjectHashSet; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.Version; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.unit.ByteSizeValue; @@ -47,8 +45,10 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Objects; +import java.util.Set; /** * ClusterInfo is an object representing a map of nodes to {@link DiskUsage} @@ -291,31 +291,32 @@ public void writeTo(StreamOutput out) throws IOException { */ public static class ReservedSpace implements Writeable { - public static final ReservedSpace EMPTY = new ReservedSpace(0, new ObjectHashSet<>()); + public static final ReservedSpace EMPTY = new ReservedSpace(0, new HashSet<>()); private final long total; - private final ObjectHashSet shardIds; + private final Set shardIds; - private ReservedSpace(long total, ObjectHashSet shardIds) { + private ReservedSpace(long total, Set shardIds) { this.total = total; - this.shardIds = shardIds; + this.shardIds = Collections.unmodifiableSet(shardIds); } ReservedSpace(StreamInput in) throws IOException { total = in.readVLong(); final int shardIdCount = in.readVInt(); - shardIds = new ObjectHashSet<>(shardIdCount); + Set shardIds = new HashSet<>(shardIdCount); for (int i = 0; i < shardIdCount; i++) { shardIds.add(new ShardId(in)); } + this.shardIds = Collections.unmodifiableSet(shardIds); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(total); out.writeVInt(shardIds.size()); - for (ObjectCursor shardIdCursor : shardIds) { - shardIdCursor.value.writeTo(out); + for (final ShardId shardIdCursor : shardIds) { + shardIdCursor.writeTo(out); } } @@ -344,8 +345,8 @@ void toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("total", total); builder.startArray("shards"); { - for (ObjectCursor shardIdCursor : shardIds) { - shardIdCursor.value.toXContent(builder, params); + for (final ShardId shardIdCursor : shardIds) { + shardIdCursor.toXContent(builder, params); } } builder.endArray(); // end "shards" @@ -358,7 +359,7 @@ void toXContent(XContentBuilder builder, Params params) throws IOException { */ public static class Builder { private long total; - private ObjectHashSet shardIds = new ObjectHashSet<>(); + private Set shardIds = new HashSet<>(); public ReservedSpace build() { assert shardIds != null : "already built"; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index b6572b15a48c9..c94f99150bd5e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.metadata; -import com.carrotsearch.hppc.LongArrayList; import org.opensearch.core.Assertions; import org.opensearch.LegacyESVersion; import org.opensearch.Version; @@ -73,6 +72,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -1796,7 +1796,7 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti } } } else if (KEY_PRIMARY_TERMS.equals(currentFieldName)) { - LongArrayList list = new LongArrayList(); + final List list = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.VALUE_NUMBER) { list.add(parser.longValue()); @@ -1804,7 +1804,7 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti throw new IllegalStateException("found a non-numeric value under [" + KEY_PRIMARY_TERMS + "]"); } } - builder.primaryTerms(list.toArray()); + builder.primaryTerms(list.stream().mapToLong(i -> i).toArray()); } else { throw new IllegalArgumentException("Unexpected field for an array " + currentFieldName); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java index 199ea21119aeb..dc985f5842224 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.metadata; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -70,7 +69,6 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.UUIDs; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Setting; @@ -81,11 +79,11 @@ import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.common.Strings; import org.opensearch.core.index.Index; -import org.opensearch.index.IndexNotFoundException; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; -import org.opensearch.core.rest.RestStatus; import org.opensearch.snapshots.RestoreService; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotsService; @@ -638,12 +636,12 @@ private void waitForShardsReadyForClosing( return; } - final ImmutableOpenIntMap shards = indexRoutingTable.getShards(); + final Map shards = indexRoutingTable.getShards(); final AtomicArray results = new AtomicArray<>(shards.size()); final CountDown countDown = new CountDown(shards.size()); - for (IntObjectCursor shard : shards) { - final IndexShardRoutingTable shardRoutingTable = shard.value; + for (final IndexShardRoutingTable shard : shards.values()) { + final IndexShardRoutingTable shardRoutingTable = shard; final int shardId = shardRoutingTable.shardId().id(); sendVerifyShardBeforeCloseRequest(shardRoutingTable, closingBlock, new NotifyOnceListener() { @Override @@ -771,12 +769,12 @@ private void waitForShardsReady( return; } - final ImmutableOpenIntMap shards = indexRoutingTable.getShards(); + final Map shards = indexRoutingTable.getShards(); final AtomicArray results = new AtomicArray<>(shards.size()); final CountDown countDown = new CountDown(shards.size()); - for (IntObjectCursor shard : shards) { - final IndexShardRoutingTable shardRoutingTable = shard.value; + for (final IndexShardRoutingTable shard : shards.values()) { + final IndexShardRoutingTable shardRoutingTable = shard; final int shardId = shardRoutingTable.shardId().id(); sendVerifyShardBlockRequest(shardRoutingTable, clusterBlock, new NotifyOnceListener() { @Override diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 224cd53126145..af348c1c98f2d 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -32,9 +32,6 @@ package org.opensearch.cluster.routing; -import com.carrotsearch.hppc.IntSet; -import com.carrotsearch.hppc.cursors.IntCursor; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.apache.lucene.util.CollectionUtil; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; @@ -47,7 +44,6 @@ import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.cluster.routing.RecoverySource.RemoteStoreRecoverySource; import org.opensearch.common.Randomness; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.Index; @@ -56,9 +52,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; @@ -86,17 +84,17 @@ public class IndexRoutingTable extends AbstractDiffable imple // note, we assume that when the index routing is created, ShardRoutings are created for all possible number of // shards with state set to UNASSIGNED - private final ImmutableOpenIntMap shards; + private final Map shards; private final List allActiveShards; - IndexRoutingTable(Index index, ImmutableOpenIntMap shards) { + IndexRoutingTable(Index index, final Map shards) { this.index = index; this.shuffler = new RotationShardShuffler(Randomness.get().nextInt()); - this.shards = shards; + this.shards = Collections.unmodifiableMap(shards); List allActiveShards = new ArrayList<>(); - for (IntObjectCursor cursor : shards) { - for (ShardRouting shardRouting : cursor.value) { + for (IndexShardRoutingTable cursor : shards.values()) { + for (ShardRouting shardRouting : cursor) { if (shardRouting.active()) { allActiveShards.add(shardRouting); } @@ -197,7 +195,7 @@ boolean validate(Metadata metadata) { @Override public Iterator iterator() { - return shards.valuesIt(); + return shards.values().iterator(); } /** @@ -232,11 +230,11 @@ public int numberOfNodesShardsAreAllocatedOn(String... excludedNodes) { return nodes.size(); } - public ImmutableOpenIntMap shards() { + public Map shards() { return shards; } - public ImmutableOpenIntMap getShards() { + public Map getShards() { return shards(); } @@ -374,7 +372,7 @@ public static Builder builder(Index index) { public static class Builder { private final Index index; - private final ImmutableOpenIntMap.Builder shards = ImmutableOpenIntMap.builder(); + private final Map shards = new HashMap<>(); public Builder(Index index) { this.index = index; @@ -418,7 +416,11 @@ public Builder initializeAsFromOpenToClose(IndexMetadata indexMetadata) { /** * Initializes a new empty index, to be restored from a snapshot */ - public Builder initializeAsNewRestore(IndexMetadata indexMetadata, SnapshotRecoverySource recoverySource, IntSet ignoreShards) { + public Builder initializeAsNewRestore( + IndexMetadata indexMetadata, + SnapshotRecoverySource recoverySource, + final Set ignoreShards + ) { final UnassignedInfo unassignedInfo = new UnassignedInfo( UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" @@ -454,7 +456,7 @@ public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, Remot "restore_source[remote_store]" ); assert indexMetadata.getIndex().equals(index); - if (!shards.isEmpty()) { + if (shards.isEmpty() == false) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardNumber = 0; shardNumber < indexMetadata.getNumberOfShards(); shardNumber++) { @@ -472,7 +474,7 @@ public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, Remot private Builder initializeAsRestore( IndexMetadata indexMetadata, RecoverySource recoverySource, - IntSet ignoreShards, + final Set ignoreShards, boolean asNew, UnassignedInfo unassignedInfo ) { @@ -550,8 +552,7 @@ private Builder initializeEmpty(IndexMetadata indexMetadata, UnassignedInfo unas } public Builder addReplica() { - for (IntCursor cursor : shards.keys()) { - int shardNumber = cursor.value; + for (final int shardNumber : shards.keySet()) { ShardId shardId = new ShardId(index, shardNumber); // version 0, will get updated when reroute will happen ShardRouting shard = ShardRouting.newUnassigned( @@ -566,8 +567,7 @@ public Builder addReplica() { } public Builder removeReplica() { - for (IntCursor cursor : shards.keys()) { - int shardId = cursor.value; + for (final int shardId : shards.keySet()) { IndexShardRoutingTable indexShard = shards.get(shardId); if (indexShard.replicaShards().isEmpty()) { // nothing to do here! @@ -621,7 +621,7 @@ public Builder addShard(ShardRouting shard) { } public IndexRoutingTable build() { - return new IndexRoutingTable(index, shards.build()); + return new IndexRoutingTable(index, shards); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java index dc9973a7ac005..5402218664f6f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.routing; -import com.carrotsearch.hppc.ObjectIntHashMap; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; import org.opensearch.core.Assertions; @@ -59,10 +58,13 @@ import java.util.ListIterator; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * {@link RoutingNodes} represents a copy the routing information contained in the {@link ClusterState cluster state}. @@ -95,7 +97,7 @@ public class RoutingNodes implements Iterable { private int relocatingShards = 0; - private final Map> nodesPerAttributeNames = new HashMap<>(); + private final Map> nodesPerAttributeNames; private final Map recoveriesPerNode = new HashMap<>(); private final Map initialReplicaRecoveries = new HashMap<>(); private final Map initialPrimaryRecoveries = new HashMap<>(); @@ -107,6 +109,7 @@ public RoutingNodes(ClusterState clusterState) { public RoutingNodes(ClusterState clusterState, boolean readOnly) { this.readOnly = readOnly; final RoutingTable routingTable = clusterState.routingTable(); + this.nodesPerAttributeNames = Collections.synchronizedMap(new HashMap<>()); // fill in the nodeToShards with the "live" nodes for (final DiscoveryNode cursor : clusterState.nodes().getDataNodes().values()) { @@ -284,18 +287,15 @@ public RoutingNode node(String nodeId) { return nodesToShards.get(nodeId); } - public ObjectIntHashMap nodesPerAttributesCounts(String attributeName) { - ObjectIntHashMap nodesPerAttributesCounts = nodesPerAttributeNames.get(attributeName); - if (nodesPerAttributesCounts != null) { - return nodesPerAttributesCounts; - } - nodesPerAttributesCounts = new ObjectIntHashMap<>(); - for (RoutingNode routingNode : this) { - String attrValue = routingNode.node().getAttributes().get(attributeName); - nodesPerAttributesCounts.addTo(attrValue, 1); - } - nodesPerAttributeNames.put(attributeName, nodesPerAttributesCounts); - return nodesPerAttributesCounts; + public Stream stream() { + return nodesToShards.values().stream(); + } + + public Set nodesPerAttributesCounts(String attributeName) { + return nodesPerAttributeNames.computeIfAbsent( + attributeName, + ignored -> stream().map(r -> r.node().getAttributes().get(attributeName)).filter(Objects::nonNull).collect(Collectors.toSet()) + ); } /** diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 23631f438b418..1bee5d8176a0f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.routing; -import com.carrotsearch.hppc.IntSet; import org.opensearch.cluster.Diff; import org.opensearch.cluster.Diffable; import org.opensearch.cluster.DiffableUtils; @@ -56,6 +55,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import static org.opensearch.cluster.metadata.MetadataIndexStateService.isIndexVerifiedBeforeClosed; @@ -578,7 +578,11 @@ public Builder addAsRestore(IndexMetadata indexMetadata, SnapshotRecoverySource return this; } - public Builder addAsNewRestore(IndexMetadata indexMetadata, SnapshotRecoverySource recoverySource, IntSet ignoreShards) { + public Builder addAsNewRestore( + IndexMetadata indexMetadata, + SnapshotRecoverySource recoverySource, + final Set ignoreShards + ) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()).initializeAsNewRestore( indexMetadata, recoverySource, diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingService.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingService.java index 9992930a1a7f6..419d2343f65cd 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingService.java @@ -8,8 +8,6 @@ package org.opensearch.cluster.routing; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -212,10 +210,10 @@ public void verifyAwarenessAttribute(String attributeName) { private void ensureWeightsSetForAllDiscoveredAndForcedAwarenessValues(ClusterState state, ClusterPutWeightedRoutingRequest request) { String attributeName = request.getWeightedRouting().attributeName(); // build attr_value -> nodes map - ObjectIntHashMap nodesPerAttribute = state.getRoutingNodes().nodesPerAttributesCounts(attributeName); + final Set nodesPerAttribute = state.getRoutingNodes().nodesPerAttributesCounts(attributeName); Set discoveredAwarenessValues = new HashSet<>(); - for (ObjectCursor stringObjectCursor : nodesPerAttribute.keys()) { - if (stringObjectCursor.value != null) discoveredAwarenessValues.add(stringObjectCursor.value); + for (String stringObjectCursor : nodesPerAttribute) { + if (stringObjectCursor != null) discoveredAwarenessValues.add(stringObjectCursor); } Set allAwarenessValues; if (forcedAwarenessAttributes.get(attributeName) == null) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java index 2351c5ec557fc..1fadd775ab7b5 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java @@ -8,7 +8,6 @@ package org.opensearch.cluster.routing.allocation.allocator; -import com.carrotsearch.hppc.ObjectIntHashMap; import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.RoutingNode; import org.opensearch.cluster.routing.RoutingNodes; @@ -25,7 +24,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -211,8 +209,8 @@ void balance() { return; } - ObjectIntHashMap nodePrimaryShardCount = calculateNodePrimaryShardCount(remoteRoutingNodes); - int totalPrimaryShardCount = Arrays.stream(nodePrimaryShardCount.values).sum(); + final Map nodePrimaryShardCount = calculateNodePrimaryShardCount(remoteRoutingNodes); + int totalPrimaryShardCount = nodePrimaryShardCount.values().stream().reduce(0, Integer::sum); totalPrimaryShardCount += routingNodes.unassigned().getNumPrimaries(); int avgPrimaryPerNode = (totalPrimaryShardCount + routingNodes.size() - 1) / routingNodes.size(); @@ -238,8 +236,8 @@ void balance() { * @param remoteRoutingNodes routing nodes for which the aggregation needs to be performed * @return map of node id to primary shard count */ - private ObjectIntHashMap calculateNodePrimaryShardCount(List remoteRoutingNodes) { - ObjectIntHashMap primaryShardCount = new ObjectIntHashMap<>(); + private Map calculateNodePrimaryShardCount(List remoteRoutingNodes) { + final Map primaryShardCount = new HashMap<>(); for (RoutingNode node : remoteRoutingNodes) { int totalPrimaryShardsPerNode = 0; for (ShardRouting shard : node) { @@ -464,7 +462,7 @@ private void tryRebalanceNode( RoutingNode sourceNode, ArrayDeque targetNodes, int avgPrimary, - ObjectIntHashMap primaryCount + final Map primaryCount ) { long shardsToBalance = primaryCount.get(sourceNode.nodeId()) - avgPrimary; assert shardsToBalance >= 0 : "Shards to balance should be greater than 0, but found negative"; @@ -493,7 +491,7 @@ private void tryRebalanceNode( if (rebalanceDecision.type() == Decision.Type.YES) { shardsToBalance--; - primaryCount.addTo(targetNode.nodeId(), 1); + primaryCount.merge(targetNode.nodeId(), 1, Integer::sum); targetNodes.offer(targetNode); break; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 811dd1249e37d..f0b79194af438 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -39,8 +39,6 @@ import java.util.Set; import java.util.function.Function; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RoutingNode; import org.opensearch.cluster.routing.ShardRouting; @@ -178,16 +176,16 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout } // build attr_value -> nodes map - ObjectIntHashMap nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute); + Set nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute); // build the count of shards per attribute value - ObjectIntHashMap shardPerAttribute = new ObjectIntHashMap<>(); + Map shardPerAttribute = new HashMap<>(); for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting.shardId())) { if (assignedShard.started() || assignedShard.initializing()) { // Note: this also counts relocation targets as that will be the new location of the shard. // Relocation sources should not be counted as the shard is moving away RoutingNode routingNode = allocation.routingNodes().node(assignedShard.currentNodeId()); - shardPerAttribute.addTo(routingNode.node().getAttributes().get(awarenessAttribute), 1); + shardPerAttribute.merge(routingNode.node().getAttributes().get(awarenessAttribute), 1, Integer::sum); } } @@ -196,15 +194,14 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId(); if (node.nodeId().equals(nodeId) == false) { // we work on different nodes, move counts around - shardPerAttribute.putOrAdd( + shardPerAttribute.compute( allocation.routingNodes().node(nodeId).node().getAttributes().get(awarenessAttribute), - 0, - -1 + (k, v) -> (v == null) ? 0 : v - 1 ); - shardPerAttribute.addTo(node.node().getAttributes().get(awarenessAttribute), 1); + shardPerAttribute.merge(node.node().getAttributes().get(awarenessAttribute), 1, Integer::sum); } } else { - shardPerAttribute.addTo(node.node().getAttributes().get(awarenessAttribute), 1); + shardPerAttribute.merge(node.node().getAttributes().get(awarenessAttribute), 1, Integer::sum); } } @@ -214,8 +211,8 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout if (fullValues != null) { // If forced awareness is enabled, numberOfAttributes = count(distinct((union(discovered_attributes, forced_attributes))) Set attributesSet = new HashSet<>(fullValues); - for (ObjectCursor stringObjectCursor : nodesPerAttribute.keys()) { - attributesSet.add(stringObjectCursor.value); + for (String stringObjectCursor : nodesPerAttribute) { + attributesSet.add(stringObjectCursor); } numberOfAttributes = attributesSet.size(); } diff --git a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java index 08e84355ab818..1f8a6aba0c883 100644 --- a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java +++ b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java @@ -32,8 +32,6 @@ package org.opensearch.common; -import com.carrotsearch.hppc.ObjectLongHashMap; -import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -41,7 +39,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import java.util.Objects; /** @@ -49,45 +49,37 @@ * * @opensearch.internal */ -public final class FieldMemoryStats implements Writeable, Iterable> { +public final class FieldMemoryStats implements Writeable, Iterable> { - private final ObjectLongHashMap stats; + private final Map stats; /** * Creates a new FieldMemoryStats instance */ - public FieldMemoryStats(ObjectLongHashMap stats) { + public FieldMemoryStats(Map stats) { this.stats = Objects.requireNonNull(stats, "status must be non-null"); - assert !stats.containsKey(null); + assert stats.containsKey(null) == false; } /** * Creates a new FieldMemoryStats instance from a stream */ public FieldMemoryStats(StreamInput input) throws IOException { - int size = input.readVInt(); - stats = new ObjectLongHashMap<>(size); - for (int i = 0; i < size; i++) { - stats.put(input.readString(), input.readVLong()); - } + stats = input.readMap(StreamInput::readString, StreamInput::readVLong); } /** * Adds / merges the given field memory stats into this stats instance */ public void add(FieldMemoryStats fieldMemoryStats) { - for (ObjectLongCursor entry : fieldMemoryStats.stats) { - stats.addTo(entry.key, entry.value); + for (final var entry : fieldMemoryStats.stats.entrySet()) { + stats.merge(entry.getKey(), entry.getValue(), Long::sum); } } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(stats.size()); - for (ObjectLongCursor entry : stats) { - out.writeString(entry.key); - out.writeVLong(entry.value); - } + out.writeMap(stats, StreamOutput::writeString, StreamOutput::writeVLong); } /** @@ -99,9 +91,9 @@ public void writeTo(StreamOutput out) throws IOException { */ public void toXContent(XContentBuilder builder, String key, String rawKey, String readableKey) throws IOException { builder.startObject(key); - for (ObjectLongCursor entry : stats) { - builder.startObject(entry.key); - builder.humanReadableField(rawKey, readableKey, new ByteSizeValue(entry.value)); + for (final var entry : stats.entrySet()) { + builder.startObject(entry.getKey()); + builder.humanReadableField(rawKey, readableKey, new ByteSizeValue(entry.getValue())); builder.endObject(); } builder.endObject(); @@ -111,7 +103,7 @@ public void toXContent(XContentBuilder builder, String key, String rawKey, Strin * Creates a deep copy of this stats instance */ public FieldMemoryStats copy() { - return new FieldMemoryStats(stats.clone()); + return new FieldMemoryStats(new HashMap<>(stats)); } @Override @@ -128,15 +120,15 @@ public int hashCode() { } @Override - public Iterator> iterator() { - return stats.iterator(); + public Iterator> iterator() { + return stats.entrySet().iterator(); } /** * Returns the fields value in bytes or 0 if it's not present in the stats */ public long get(String field) { - return stats.get(field); + return stats.getOrDefault(field, 0L); } /** diff --git a/server/src/main/java/org/opensearch/common/collect/HppcMaps.java b/server/src/main/java/org/opensearch/common/collect/HppcMaps.java deleted file mode 100644 index 2fd7316accf6e..0000000000000 --- a/server/src/main/java/org/opensearch/common/collect/HppcMaps.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.collect; - -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.ObjectLookupContainer; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.cursors.ObjectCursor; - -import java.util.Iterator; - -/** - * High performance maps - * - * @opensearch.internal - */ -public final class HppcMaps { - - private HppcMaps() {} - - /** - * Returns a new map with the given number of expected elements. - * - * @param expectedElements - * The expected number of elements guaranteed not to cause buffer - * expansion (inclusive). - */ - public static ObjectObjectHashMap newMap(int expectedElements) { - return new ObjectObjectHashMap<>(expectedElements); - } - - /** - * Returns a new map with a default initial capacity. - */ - public static ObjectObjectHashMap newMap() { - return newMap(16); - } - - /** - * Returns a map like {@link #newMap()} that does not accept null keys - */ - public static ObjectObjectHashMap newNoNullKeysMap() { - return ensureNoNullKeys(16); - } - - /** - * Returns a map like {@link #newMap(int)} that does not accept null keys - * - * @param expectedElements - * The expected number of elements guaranteed not to cause buffer - * expansion (inclusive). - */ - public static ObjectObjectHashMap newNoNullKeysMap(int expectedElements) { - return ensureNoNullKeys(expectedElements); - } - - /** - * Wraps the given map and prevent adding of null keys. - * - * @param expectedElements - * The expected number of elements guaranteed not to cause buffer - * expansion (inclusive). - */ - public static ObjectObjectHashMap ensureNoNullKeys(int expectedElements) { - return new ObjectObjectHashMap(expectedElements) { - @Override - public V put(K key, V value) { - if (key == null) { - throw new IllegalArgumentException("Map key must not be null"); - } - return super.put(key, value); - } - }; - } - - /** - * @return an intersection view over the two specified containers (which can be KeyContainer or ObjectHashSet). - */ - // Hppc has forEach, but this means we need to build an intermediate set, with this method we just iterate - // over each unique value without creating a third set. - public static Iterable intersection(ObjectLookupContainer container1, final ObjectLookupContainer container2) { - assert container1 != null && container2 != null; - final Iterator> iterator = container1.iterator(); - final Iterator intersection = new Iterator() { - - T current; - - @Override - public boolean hasNext() { - if (iterator.hasNext()) { - do { - T next = iterator.next().value; - if (container2.contains(next)) { - current = next; - return true; - } - } while (iterator.hasNext()); - } - return false; - } - - @Override - public T next() { - return current; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - return new Iterable() { - @Override - public Iterator iterator() { - return intersection; - } - }; - } - - /** - * Object for the map - * - * @opensearch.internal - */ - public static final class Object { - /** - * Integer type for the map - * - * @opensearch.internal - */ - public static final class Integer { - public static ObjectIntHashMap ensureNoNullKeys(int capacity, float loadFactor) { - return new ObjectIntHashMap(capacity, loadFactor) { - @Override - public int put(V key, int value) { - if (key == null) { - throw new IllegalArgumentException("Map key must not be null"); - } - return super.put(key, value); - } - }; - } - } - } -} diff --git a/server/src/main/java/org/opensearch/common/collect/ImmutableOpenIntMap.java b/server/src/main/java/org/opensearch/common/collect/ImmutableOpenIntMap.java deleted file mode 100644 index 0bff76e7ec90e..0000000000000 --- a/server/src/main/java/org/opensearch/common/collect/ImmutableOpenIntMap.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.collect; - -import com.carrotsearch.hppc.IntCollection; -import com.carrotsearch.hppc.IntContainer; -import com.carrotsearch.hppc.IntLookupContainer; -import com.carrotsearch.hppc.IntObjectAssociativeContainer; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.IntObjectMap; -import com.carrotsearch.hppc.ObjectContainer; -import com.carrotsearch.hppc.cursors.IntCursor; -import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.predicates.IntObjectPredicate; -import com.carrotsearch.hppc.predicates.IntPredicate; -import com.carrotsearch.hppc.procedures.IntObjectProcedure; - -import java.util.Iterator; -import java.util.Map; - -/** - * An immutable map implementation based on open hash map. - *

    - * Can be constructed using a {@link #builder()}, or using {@link #builder(org.opensearch.common.collect.ImmutableOpenIntMap)} - * (which is an optimized option to copy over existing content and modify it). - * - * @opensearch.internal - */ -public final class ImmutableOpenIntMap implements Iterable> { - - private final IntObjectHashMap map; - - private ImmutableOpenIntMap(IntObjectHashMap map) { - this.map = map; - } - - /** - * @return Returns the value associated with the given key or the default value - * for the key type, if the key is not associated with any value. - *

    - * Important note: For primitive type values, the value returned for a non-existing - * key may not be the default value of the primitive type (it may be any value previously - * assigned to that slot). - */ - public VType get(int key) { - return map.get(key); - } - - /** - * Returns true if this container has an association to a value for - * the given key. - */ - public boolean containsKey(int key) { - return map.containsKey(key); - } - - /** - * @return Returns the current size (number of assigned keys) in the container. - */ - public int size() { - return map.size(); - } - - /** - * @return Return true if this hash map contains no assigned keys. - */ - public boolean isEmpty() { - return map.isEmpty(); - } - - /** - * Returns a cursor over the entries (key-value pairs) in this map. The iterator is - * implemented as a cursor and it returns the same cursor instance on every - * call to {@link java.util.Iterator#next()}. To read the current key and value use the cursor's - * public fields. An example is shown below. - * 

    -     * for (IntShortCursor c : intShortMap)
-     * {
-     *     System.out.println("index=" + c.index
-     *       + " key=" + c.key
-     *       + " value=" + c.value);
-     * }
-     *
    - *

    - * The index field inside the cursor gives the internal index inside - * the container's implementation. The interpretation of this index depends on - * to the container. - */ - @Override - public Iterator> iterator() { - return map.iterator(); - } - - /** - * Returns a specialized view of the keys of this associated container. - * The view additionally implements {@link com.carrotsearch.hppc.ObjectLookupContainer}. - */ - public IntLookupContainer keys() { - return map.keys(); - } - - /** - * Returns a direct iterator over the keys. - */ - public Iterator keysIt() { - final Iterator iterator = map.keys().iterator(); - return new Iterator() { - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public Integer next() { - return iterator.next().value; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - /** - * @return Returns a container with all values stored in this map. - */ - public ObjectContainer values() { - return map.values(); - } - - /** - * Returns a direct iterator over the keys. - */ - public Iterator valuesIt() { - final Iterator> iterator = map.values().iterator(); - return new Iterator() { - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public VType next() { - return iterator.next().value; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - @Override - public String toString() { - return map.toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ImmutableOpenIntMap that = (ImmutableOpenIntMap) o; - - if (!map.equals(that.map)) return false; - - return true; - } - - @Override - public int hashCode() { - return map.hashCode(); - } - - @SuppressWarnings("unchecked") - private static final ImmutableOpenIntMap EMPTY = new ImmutableOpenIntMap(new IntObjectHashMap()); - - @SuppressWarnings("unchecked") - public static ImmutableOpenIntMap of() { - return EMPTY; - } - - public static Builder builder() { - return new Builder<>(); - } - - public static Builder builder(int size) { - return new Builder<>(size); - } - - public static Builder builder(ImmutableOpenIntMap map) { - return new Builder<>(map); - } - - /** - * Base builder for an immutable int - * - * @opensearch.internal - */ - public static class Builder implements IntObjectMap { - - private IntObjectHashMap map; - - public Builder() { - // noinspection unchecked - this(EMPTY); - } - - public Builder(int size) { - this.map = new IntObjectHashMap<>(size); - } - - public Builder(ImmutableOpenIntMap map) { - this.map = map.map.clone(); - } - - /** - * Builds a new instance of the - */ - public ImmutableOpenIntMap build() { - IntObjectHashMap map = this.map; - this.map = null; // nullify the map, so any operation post build will fail! (hackish, but safest) - return new ImmutableOpenIntMap<>(map); - } - - /** - * Puts all the entries in the map to the builder. - */ - public Builder putAll(Map map) { - for (Map.Entry entry : map.entrySet()) { - this.map.put(entry.getKey(), entry.getValue()); - } - return this; - } - - /** - * A put operation that can be used in the fluent pattern. - */ - public Builder fPut(int key, VType value) { - map.put(key, value); - return this; - } - - @Override - public VType put(int key, VType value) { - return map.put(key, value); - } - - @Override - public VType get(int key) { - return map.get(key); - } - - @Override - public VType getOrDefault(int kType, VType vType) { - return map.getOrDefault(kType, vType); - } - - /** - * Remove that can be used in the fluent pattern. - */ - public Builder fRemove(int key) { - map.remove(key); - return this; - } - - @Override - public VType remove(int key) { - return map.remove(key); - } - - @Override - public Iterator> iterator() { - return map.iterator(); - } - - @Override - public boolean containsKey(int key) { - return map.containsKey(key); - } - - @Override - public int size() { - return map.size(); - } - - @Override - public boolean isEmpty() { - return map.isEmpty(); - } - - @Override - public void clear() { - map.clear(); - } - - @Override - public int putAll(IntObjectAssociativeContainer container) { - return map.putAll(container); - } - - @Override - public int putAll(Iterable> iterable) { - return map.putAll(iterable); - } - - @Override - public int removeAll(IntContainer container) { - return map.removeAll(container); - } - - @Override - public int removeAll(IntPredicate predicate) { - return map.removeAll(predicate); - } - - @Override - public > T forEach(T procedure) { - return map.forEach(procedure); - } - - @Override - public IntCollection keys() { - return map.keys(); - } - - @Override - public ObjectContainer values() { - return map.values(); - } - - @Override - public int removeAll(IntObjectPredicate predicate) { - return map.removeAll(predicate); - } - - @Override - public > T forEach(T predicate) { - return map.forEach(predicate); - } - - @Override - public int indexOf(int key) { - return map.indexOf(key); - } - - @Override - public boolean indexExists(int index) { - return map.indexExists(index); - } - - @Override - public VType indexGet(int index) { - return map.indexGet(index); - } - - @Override - public VType indexReplace(int index, VType newValue) { - return map.indexReplace(index, newValue); - } - - @Override - public void indexInsert(int index, int key, VType value) { - map.indexInsert(index, key, value); - } - - @Override - public void release() { - map.release(); - } - - @Override - public String visualizeKeyDistribution(int characters) { - return map.visualizeKeyDistribution(characters); - } - } -} diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java index a1ab761ec8034..98ebb34fc040e 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -32,7 +32,6 @@ package org.opensearch.common.lucene.search; -import com.carrotsearch.hppc.ObjectHashSet; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; @@ -50,10 +49,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Objects; +import java.util.Set; /** * A multi phrase prefix query. @@ -174,7 +175,7 @@ public Query rewrite(IndexReader reader) throws IOException { } Term[] suffixTerms = termArrays.get(sizeMinus1); int position = positions.get(sizeMinus1); - ObjectHashSet terms = new ObjectHashSet<>(); + Set terms = new HashSet<>(); for (Term term : suffixTerms) { getPrefixTerms(terms, term, reader); if (terms.size() > maxExpansions) { @@ -196,11 +197,11 @@ public Query rewrite(IndexReader reader) throws IOException { ) .build(); } - query.add(terms.toArray(Term.class), position); + query.add(terms.toArray(new Term[0]), position); return query.build(); } - private void getPrefixTerms(ObjectHashSet terms, final Term prefix, final IndexReader reader) throws IOException { + private void getPrefixTerms(final Set terms, final Term prefix, final IndexReader reader) throws IOException { // SlowCompositeReaderWrapper could be used... but this would merge all terms from each segment into one terms // instance, which is very expensive. Therefore I think it is better to iterate over each leaf individually. List leaves = reader.leaves(); diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/RandomScoreFunction.java b/server/src/main/java/org/opensearch/common/lucene/search/function/RandomScoreFunction.java index 48b3e0d04bdc3..b743a7bd3e707 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/RandomScoreFunction.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/RandomScoreFunction.java @@ -31,11 +31,11 @@ package org.opensearch.common.lucene.search.function; -import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; import org.apache.lucene.util.StringHelper; import org.opensearch.common.Nullable; +import org.opensearch.common.util.BitMixer; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.SortedBinaryDocValues; diff --git a/server/src/main/java/org/opensearch/common/recycler/Recyclers.java b/server/src/main/java/org/opensearch/common/recycler/Recyclers.java index 74916cf4e6d10..4cbb80509d6a1 100644 --- a/server/src/main/java/org/opensearch/common/recycler/Recyclers.java +++ b/server/src/main/java/org/opensearch/common/recycler/Recyclers.java @@ -32,7 +32,7 @@ package org.opensearch.common.recycler; -import com.carrotsearch.hppc.BitMixer; +import org.opensearch.common.util.BitMixer; import java.util.ArrayDeque; diff --git a/server/src/main/java/org/opensearch/common/util/AbstractPagedHashMap.java b/server/src/main/java/org/opensearch/common/util/AbstractPagedHashMap.java index 1ff3038297008..450dcc833a7b3 100644 --- a/server/src/main/java/org/opensearch/common/util/AbstractPagedHashMap.java +++ b/server/src/main/java/org/opensearch/common/util/AbstractPagedHashMap.java @@ -32,7 +32,6 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.BitMixer; import org.opensearch.common.lease.Releasable; /** diff --git a/server/src/main/java/org/opensearch/common/util/BytesRefHash.java b/server/src/main/java/org/opensearch/common/util/BytesRefHash.java index da11d91b1f535..ecc93d017beaf 100644 --- a/server/src/main/java/org/opensearch/common/util/BytesRefHash.java +++ b/server/src/main/java/org/opensearch/common/util/BytesRefHash.java @@ -32,7 +32,6 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.util.BytesRef; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/common/util/LongLongHash.java b/server/src/main/java/org/opensearch/common/util/LongLongHash.java index 1a720eae82a1d..f1cdd29932b2f 100644 --- a/server/src/main/java/org/opensearch/common/util/LongLongHash.java +++ b/server/src/main/java/org/opensearch/common/util/LongLongHash.java @@ -32,8 +32,6 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.BitMixer; - import org.opensearch.common.lease.Releasables; /** diff --git a/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java b/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java index 5789b47423c1d..417eb6a316d86 100644 --- a/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java +++ b/server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java @@ -8,7 +8,6 @@ package org.opensearch.common.util; -import org.apache.lucene.util.hppc.BitMixer; import org.opensearch.common.lease.Releasable; /** diff --git a/server/src/main/java/org/opensearch/gateway/Gateway.java b/server/src/main/java/org/opensearch/gateway/Gateway.java index 15b226e2d30e2..a01f053ce6e9b 100644 --- a/server/src/main/java/org/opensearch/gateway/Gateway.java +++ b/server/src/main/java/org/opensearch/gateway/Gateway.java @@ -32,7 +32,6 @@ package org.opensearch.gateway; -import com.carrotsearch.hppc.ObjectFloatHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.FailedNodeException; @@ -44,6 +43,8 @@ import org.opensearch.core.index.Index; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.function.Function; /** @@ -81,7 +82,7 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t } } - final ObjectFloatHashMap indices = new ObjectFloatHashMap<>(); + final Map indices = new HashMap<>(); Metadata electedGlobalState = null; int found = 0; for (final TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) { @@ -95,7 +96,7 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t electedGlobalState = nodeState.metadata(); } for (final IndexMetadata cursor : nodeState.metadata().indices().values()) { - indices.addTo(cursor.getIndex(), 1); + indices.merge(cursor.getIndex(), 1f, Float::sum); } } if (found < requiredAllocation) { @@ -106,7 +107,7 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t final Metadata.Builder metadataBuilder = Metadata.builder(electedGlobalState).removeAllIndices(); assert !indices.containsKey(null); - final Object[] keys = indices.keys; + final Object[] keys = indices.keySet().toArray(); for (int i = 0; i < keys.length; i++) { if (keys[i] != null) { final Index index = (Index) keys[i]; diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 8c3b5b6ceecbd..46f296f52ae01 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -32,8 +32,6 @@ package org.opensearch.http; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -66,6 +64,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -264,12 +263,12 @@ static int resolvePublishPort(Settings settings, List boundAdd // if no matching boundAddress found, check if there is a unique port for all bound addresses if (publishPort < 0) { - final IntSet ports = new IntHashSet(); + final Set ports = new HashSet<>(); for (TransportAddress boundAddress : boundAddresses) { ports.add(boundAddress.getPort()); } if (ports.size() == 1) { - publishPort = ports.iterator().next().value; + publishPort = ports.iterator().next(); } } diff --git a/server/src/main/java/org/opensearch/index/engine/CombinedDeletionPolicy.java b/server/src/main/java/org/opensearch/index/engine/CombinedDeletionPolicy.java index 528bd556681cc..4d7faf6c9e375 100644 --- a/server/src/main/java/org/opensearch/index/engine/CombinedDeletionPolicy.java +++ b/server/src/main/java/org/opensearch/index/engine/CombinedDeletionPolicy.java @@ -32,7 +32,6 @@ package org.opensearch.index.engine; -import com.carrotsearch.hppc.ObjectIntHashMap; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; @@ -45,6 +44,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,7 +64,7 @@ public class CombinedDeletionPolicy extends IndexDeletionPolicy { private final TranslogDeletionPolicy translogDeletionPolicy; private final SoftDeletesPolicy softDeletesPolicy; private final LongSupplier globalCheckpointSupplier; - private final ObjectIntHashMap snapshottedCommits; // Number of snapshots held against each commit point. + private final Map snapshottedCommits; // Number of snapshots held against each commit point. private volatile IndexCommit safeCommit; // the most recent safe commit point - its max_seqno at most the persisted global checkpoint. private volatile long maxSeqNoOfNextSafeCommit; private volatile IndexCommit lastCommit; // the most recent commit point @@ -80,7 +80,7 @@ public class CombinedDeletionPolicy extends IndexDeletionPolicy { this.translogDeletionPolicy = translogDeletionPolicy; this.softDeletesPolicy = softDeletesPolicy; this.globalCheckpointSupplier = globalCheckpointSupplier; - this.snapshottedCommits = new ObjectIntHashMap<>(); + this.snapshottedCommits = new HashMap<>(); } @Override @@ -174,7 +174,7 @@ synchronized IndexCommit acquireIndexCommit(boolean acquiringSafeCommit) { assert safeCommit != null : "Safe commit is not initialized yet"; assert lastCommit != null : "Last commit is not initialized yet"; final IndexCommit snapshotting = acquiringSafeCommit ? safeCommit : lastCommit; - snapshottedCommits.addTo(snapshotting, 1); // increase refCount + snapshottedCommits.merge(snapshotting, 1, Integer::sum); // increase refCount return new SnapshotIndexCommit(snapshotting); } @@ -191,7 +191,7 @@ synchronized boolean releaseCommit(final IndexCommit snapshotCommit) { + "], releasing commit [" + releasingCommit + "]"; - final int refCount = snapshottedCommits.addTo(releasingCommit, -1); // release refCount + final int refCount = snapshottedCommits.merge(releasingCommit, -1, Integer::sum); // release refCount assert refCount >= 0 : "Number of snapshots can not be negative [" + refCount + "]"; if (refCount == 0) { snapshottedCommits.remove(releasingCommit); diff --git a/server/src/main/java/org/opensearch/index/engine/CompletionStatsCache.java b/server/src/main/java/org/opensearch/index/engine/CompletionStatsCache.java index 08ede10f3f6aa..c6cf7e8c1b53f 100644 --- a/server/src/main/java/org/opensearch/index/engine/CompletionStatsCache.java +++ b/server/src/main/java/org/opensearch/index/engine/CompletionStatsCache.java @@ -31,8 +31,6 @@ package org.opensearch.index.engine; -import com.carrotsearch.hppc.ObjectLongHashMap; -import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -47,6 +45,8 @@ import org.opensearch.common.util.CollectionUtils; import org.opensearch.search.suggest.completion.CompletionStats; +import java.util.HashMap; +import java.util.Map; import java.util.function.Supplier; /** @@ -99,7 +99,7 @@ CompletionStats get(String... fieldNamePatterns) { // we won the race, nobody else is already computing stats, so it's up to us ActionListener.completeWith(newFuture, () -> { long sizeInBytes = 0; - final ObjectLongHashMap completionFields = new ObjectLongHashMap<>(); + final Map completionFields = new HashMap<>(); try (Engine.Searcher currentSearcher = searcherSupplier.get()) { for (LeafReaderContext atomicReaderContext : currentSearcher.getIndexReader().leaves()) { @@ -109,7 +109,7 @@ CompletionStats get(String... fieldNamePatterns) { if (terms instanceof CompletionTerms) { // TODO: currently we load up the suggester for reporting its size final long fstSize = ((CompletionTerms) terms).suggester().ramBytesUsed(); - completionFields.addTo(info.name, fstSize); + completionFields.merge(info.name, fstSize, Long::sum); sizeInBytes += fstSize; } } @@ -143,10 +143,10 @@ CompletionStats get(String... fieldNamePatterns) { private static CompletionStats filterCompletionStatsByFieldName(String[] fieldNamePatterns, CompletionStats fullCompletionStats) { final FieldMemoryStats fieldMemoryStats; if (CollectionUtils.isEmpty(fieldNamePatterns) == false) { - final ObjectLongHashMap completionFields = new ObjectLongHashMap<>(fieldNamePatterns.length); - for (ObjectLongCursor fieldCursor : fullCompletionStats.getFields()) { - if (Regex.simpleMatch(fieldNamePatterns, fieldCursor.key)) { - completionFields.addTo(fieldCursor.key, fieldCursor.value); + final Map completionFields = new HashMap<>(fieldNamePatterns.length); + for (var fieldCursor : fullCompletionStats.getFields()) { + if (Regex.simpleMatch(fieldNamePatterns, fieldCursor.getKey())) { + completionFields.merge(fieldCursor.getKey(), fieldCursor.getValue(), Long::sum); } } fieldMemoryStats = new FieldMemoryStats(completionFields); diff --git a/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java index f03896c4b670a..138b417571784 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java @@ -32,7 +32,6 @@ package org.opensearch.index.fielddata; -import com.carrotsearch.hppc.ObjectLongHashMap; import org.apache.lucene.util.Accountable; import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.metrics.CounterMetric; @@ -41,6 +40,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.index.shard.ShardId; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentMap; @@ -56,9 +56,9 @@ public class ShardFieldData implements IndexFieldDataCache.Listener { private final ConcurrentMap perFieldTotals = ConcurrentCollections.newConcurrentMap(); public FieldDataStats stats(String... fields) { - ObjectLongHashMap fieldTotals = null; + Map fieldTotals = null; if (CollectionUtils.isEmpty(fields) == false) { - fieldTotals = new ObjectLongHashMap<>(); + fieldTotals = new HashMap<>(); for (Map.Entry entry : perFieldTotals.entrySet()) { if (Regex.simpleMatch(fields, entry.getKey())) { fieldTotals.put(entry.getKey(), entry.getValue().count()); diff --git a/server/src/main/java/org/opensearch/index/mapper/ParseContext.java b/server/src/main/java/org/opensearch/index/mapper/ParseContext.java index 96fd0490ae374..092e2a6fd3710 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParseContext.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParseContext.java @@ -32,8 +32,6 @@ package org.opensearch.index.mapper; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.ObjectObjectMap; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; @@ -44,10 +42,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -68,7 +68,7 @@ public static class Document implements Iterable { private final String path; private final String prefix; private final List fields; - private ObjectObjectMap keyedFields; + private Map keyedFields; private Document(String path, Document parent) { fields = new ArrayList<>(); @@ -124,7 +124,7 @@ public void add(IndexableField field) { /** Add fields so that they can later be fetched using {@link #getByKey(Object)}. */ public void addWithKey(Object key, IndexableField field) { if (keyedFields == null) { - keyedFields = new ObjectObjectHashMap<>(); + keyedFields = new HashMap<>(); } else if (keyedFields.containsKey(key)) { throw new IllegalStateException("Only one field can be stored per key"); } diff --git a/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java b/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java index d75893080c0d7..0a4c197898d3d 100644 --- a/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java @@ -32,10 +32,11 @@ package org.opensearch.index.seqno; -import com.carrotsearch.hppc.LongObjectHashMap; import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; /** @@ -56,13 +57,13 @@ public class LocalCheckpointTracker { * A collection of bit sets representing processed sequence numbers. Each sequence number is mapped to a bit set by dividing by the * bit set size. */ - final LongObjectHashMap processedSeqNo = new LongObjectHashMap<>(); + final Map processedSeqNo = new HashMap<>(); /** * A collection of bit sets representing durably persisted sequence numbers. Each sequence number is mapped to a bit set by dividing by * the bit set size. */ - final LongObjectHashMap persistedSeqNo = new LongObjectHashMap<>(); + final Map persistedSeqNo = new HashMap<>(); /** * The current local checkpoint, i.e., all sequence numbers no more than this number have been processed. @@ -162,7 +163,7 @@ public synchronized void fastForwardProcessedSeqNo(final long seqNo) { processedCheckpoint.compareAndSet(currentProcessedCheckpoint, seqNo); } - private void markSeqNo(final long seqNo, final AtomicLong checkPoint, final LongObjectHashMap bitSetMap) { + private void markSeqNo(final long seqNo, final AtomicLong checkPoint, final Map bitSetMap) { assert Thread.holdsLock(this); // make sure we track highest seen sequence number advanceMaxSeqNo(seqNo); @@ -256,7 +257,7 @@ public boolean hasProcessed(final long seqNo) { * following the current checkpoint is processed. */ @SuppressForbidden(reason = "Object#notifyAll") - private void updateCheckpoint(AtomicLong checkPoint, LongObjectHashMap bitSetMap) { + private void updateCheckpoint(AtomicLong checkPoint, final Map bitSetMap) { assert Thread.holdsLock(this); assert getBitSetForSeqNo(bitSetMap, checkPoint.get() + 1).get(seqNoToBitSetOffset(checkPoint.get() + 1)) : "updateCheckpoint is called but the bit following the checkpoint is not set"; @@ -302,23 +303,15 @@ private static long getBitSetKey(final long seqNo) { return seqNo / BIT_SET_SIZE; } - private CountedBitSet getBitSetForSeqNo(final LongObjectHashMap bitSetMap, final long seqNo) { + private CountedBitSet getBitSetForSeqNo(final Map bitSetMap, final long seqNo) { assert Thread.holdsLock(this); final long bitSetKey = getBitSetKey(seqNo); - final int index = bitSetMap.indexOf(bitSetKey); - final CountedBitSet bitSet; - if (bitSetMap.indexExists(index)) { - bitSet = bitSetMap.indexGet(index); - } else { - bitSet = new CountedBitSet(BIT_SET_SIZE); - bitSetMap.indexInsert(index, bitSetKey, bitSet); - } - return bitSet; + return bitSetMap.computeIfAbsent(bitSetKey, k -> new CountedBitSet(BIT_SET_SIZE)); } /** * Obtain the position in the bit set corresponding to the provided sequence number. The bit set corresponding to the sequence number - * can be obtained via {@link #getBitSetForSeqNo(LongObjectHashMap, long)}. + * can be obtained via {@link #getBitSetForSeqNo(Map, long)}. * * @param seqNo the sequence number to obtain the position for * @return the position in the bit set corresponding to the provided sequence number diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index 08e899a40af7f..4d6cffa58510f 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -32,8 +32,6 @@ package org.opensearch.index.seqno; -import com.carrotsearch.hppc.ObjectLongHashMap; -import com.carrotsearch.hppc.ObjectLongMap; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.LegacyESVersion; import org.opensearch.Version; @@ -816,10 +814,10 @@ public int hashCode() { * * @return a map from allocation ID to the local knowledge of the persisted global checkpoint for that allocation ID */ - public synchronized ObjectLongMap getInSyncGlobalCheckpoints() { + public synchronized Map getInSyncGlobalCheckpoints() { assert primaryMode; assert handoffInProgress == false; - final ObjectLongMap globalCheckpoints = new ObjectLongHashMap<>(checkpoints.size()); // upper bound on the size + final Map globalCheckpoints = new HashMap<>(checkpoints.size()); // upper bound on the size checkpoints.entrySet() .stream() .filter(e -> e.getValue().inSync && e.getValue().replicated) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 5deaf5f9a483a..0eede0cdc4c8d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -32,7 +32,6 @@ package org.opensearch.index.shard; -import com.carrotsearch.hppc.ObjectLongMap; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; @@ -3164,7 +3163,7 @@ public long getLastSyncedGlobalCheckpoint() { * * @return a map from allocation ID to the local knowledge of the global checkpoint for that allocation ID */ - public ObjectLongMap getInSyncGlobalCheckpoints() { + public Map getInSyncGlobalCheckpoints() { assert assertPrimaryMode(); verifyNotClosed(); return replicationTracker.getInSyncGlobalCheckpoints(); @@ -3185,7 +3184,7 @@ public void maybeSyncGlobalCheckpoint(final String reason) { final SeqNoStats stats = getEngine().getSeqNoStats(replicationTracker.getGlobalCheckpoint()); final boolean asyncDurability = indexSettings().getTranslogDurability() == Durability.ASYNC; if (stats.getMaxSeqNo() == stats.getGlobalCheckpoint() || asyncDurability) { - final ObjectLongMap globalCheckpoints = getInSyncGlobalCheckpoints(); + final Map globalCheckpoints = getInSyncGlobalCheckpoints(); final long globalCheckpoint = replicationTracker.getGlobalCheckpoint(); // async durability means that the local checkpoint might lag (as it is only advanced on fsync) // periodically ask for the newest local checkpoint by syncing the global checkpoint, so that ultimately the global @@ -3195,7 +3194,7 @@ public void maybeSyncGlobalCheckpoint(final String reason) { final boolean syncNeeded = (asyncDurability && (stats.getGlobalCheckpoint() < stats.getMaxSeqNo() || replicationTracker.pendingInSync())) // check if the persisted global checkpoint - || StreamSupport.stream(globalCheckpoints.values().spliterator(), false).anyMatch(v -> v.value < globalCheckpoint); + || StreamSupport.stream(globalCheckpoints.values().spliterator(), false).anyMatch(v -> v < globalCheckpoint); // only sync if index is not closed and there is a shard lagging the primary if (syncNeeded && indexSettings.getIndexMetadata().getState() == IndexMetadata.State.OPEN) { logger.trace("syncing global checkpoint for [{}]", reason); diff --git a/server/src/main/java/org/opensearch/index/translog/MultiSnapshot.java b/server/src/main/java/org/opensearch/index/translog/MultiSnapshot.java index bda8bbecb81f0..941283afe5908 100644 --- a/server/src/main/java/org/opensearch/index/translog/MultiSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/MultiSnapshot.java @@ -32,13 +32,14 @@ package org.opensearch.index.translog; -import com.carrotsearch.hppc.LongObjectHashMap; import org.opensearch.index.seqno.CountedBitSet; import org.opensearch.index.seqno.SequenceNumbers; import java.io.Closeable; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; /** * A snapshot composed out of multiple snapshots @@ -105,7 +106,7 @@ public void close() throws IOException { */ static final class SeqNoSet { static final short BIT_SET_SIZE = 1024; - private final LongObjectHashMap bitSets = new LongObjectHashMap<>(); + private final Map bitSets = new HashMap<>(); /** * Marks this sequence number and returns {@code true} if it is seen before. diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 1fd3a9c3341f6..dd85f8f1f77ea 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -32,8 +32,6 @@ package org.opensearch.index.translog; -import com.carrotsearch.hppc.LongArrayList; -import com.carrotsearch.hppc.procedures.LongProcedure; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; @@ -62,7 +60,9 @@ import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -105,7 +105,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable { // lock order synchronized(syncLock) -> try(Releasable lock = writeLock.acquire()) -> synchronized(this) private final Object syncLock = new Object(); - private LongArrayList nonFsyncedSequenceNumbers = new LongArrayList(64); + private List nonFsyncedSequenceNumbers = new ArrayList<>(64); private final int forceWriteThreshold; private volatile long bufferedBytes; private ReleasableBytesStreamOutput buffer; @@ -511,7 +511,7 @@ final boolean syncUpTo(long offset) throws IOException { // double checked locking - we don't want to fsync unless we have to and now that we have // the lock we should check again since if this code is busy we might have fsynced enough already final Checkpoint checkpointToSync; - final LongArrayList flushedSequenceNumbers; + final List flushedSequenceNumbers; final ReleasableBytesReference toWrite; try (ReleasableLock toClose = writeLock.acquire()) { synchronized (this) { @@ -519,7 +519,7 @@ final boolean syncUpTo(long offset) throws IOException { checkpointToSync = getCheckpoint(); toWrite = pollOpsToWrite(); flushedSequenceNumbers = nonFsyncedSequenceNumbers; - nonFsyncedSequenceNumbers = new LongArrayList(64); + nonFsyncedSequenceNumbers = new ArrayList<>(64); } try { @@ -541,7 +541,7 @@ final boolean syncUpTo(long offset) throws IOException { closeWithTragicEvent(ex); throw ex; } - flushedSequenceNumbers.forEach((LongProcedure) persistedSequenceNumberConsumer::accept); + flushedSequenceNumbers.forEach(persistedSequenceNumberConsumer::accept); assert lastSyncedCheckpoint.offset <= checkpointToSync.offset : "illegal state: " + lastSyncedCheckpoint.offset + " <= " diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index a0208fab9cbef..c06880db42587 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -32,8 +32,6 @@ package org.opensearch.indices; -import com.carrotsearch.hppc.ObjectHashSet; -import com.carrotsearch.hppc.ObjectSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; @@ -59,6 +57,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.Objects; import java.util.Set; @@ -330,8 +329,8 @@ public int hashCode() { } synchronized void cleanCache() { - final ObjectSet currentKeysToClean = new ObjectHashSet<>(); - final ObjectSet currentFullClean = new ObjectHashSet<>(); + final Set currentKeysToClean = new HashSet<>(); + final Set currentFullClean = new HashSet<>(); currentKeysToClean.clear(); currentFullClean.clear(); for (Iterator iterator = keysToClean.iterator(); iterator.hasNext();) { diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java index f59241fad3a54..702ffff1c9330 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java @@ -32,7 +32,6 @@ package org.opensearch.rest.action.cat; -import com.carrotsearch.hppc.ObjectIntScatterMap; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -51,7 +50,9 @@ import org.opensearch.rest.action.RestActionListener; import org.opensearch.rest.action.RestResponseListener; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; @@ -131,7 +132,7 @@ protected Table getTableWithHeader(final RestRequest request) { } private Table buildTable(RestRequest request, final ClusterStateResponse state, final NodesStatsResponse stats) { - final ObjectIntScatterMap allocs = new ObjectIntScatterMap<>(); + final Map allocs = new HashMap<>(); for (ShardRouting shard : state.getState().routingTable().allShards()) { String nodeId = "UNASSIGNED"; @@ -140,7 +141,7 @@ private Table buildTable(RestRequest request, final ClusterStateResponse state, nodeId = shard.currentNodeId(); } - allocs.addTo(nodeId, 1); + allocs.merge(nodeId, 1, Integer::sum); } Table table = getTableWithHeader(request); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java index df70ee5943aad..a04bac0c30bc9 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java @@ -32,7 +32,6 @@ package org.opensearch.rest.action.cat; -import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -107,14 +106,14 @@ private Table buildTable(final RestRequest request, final NodesStatsResponse nod for (NodeStats nodeStats : nodeStatses.getNodes()) { if (nodeStats.getIndices().getFieldData().getFields() != null) { - for (ObjectLongCursor cursor : nodeStats.getIndices().getFieldData().getFields()) { + for (var cursor : nodeStats.getIndices().getFieldData().getFields()) { table.startRow(); table.addCell(nodeStats.getNode().getId()); table.addCell(nodeStats.getNode().getHostName()); table.addCell(nodeStats.getNode().getHostAddress()); table.addCell(nodeStats.getNode().getName()); - table.addCell(cursor.key); - table.addCell(new ByteSizeValue(cursor.value)); + table.addCell(cursor.getKey()); + table.addCell(new ByteSizeValue(cursor.getValue())); table.endRow(); } } diff --git a/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java b/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java index f955413907714..b94ff77a1d0b7 100644 --- a/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java +++ b/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java @@ -46,7 +46,7 @@ import java.time.ZoneId; -import static com.carrotsearch.hppc.BitMixer.mix32; +import static org.opensearch.common.util.BitMixer.mix32; /** * Utilities for scoring scripts diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java index 25f0fa7bc9b3e..2d517a1220e98 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -31,7 +31,6 @@ package org.opensearch.search.aggregations.bucket.nested; -import com.carrotsearch.hppc.LongArrayList; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; @@ -58,6 +57,8 @@ import org.opensearch.search.internal.SearchContext; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; /** @@ -169,7 +170,7 @@ class BufferingNestedLeafBucketCollector extends LeafBucketCollectorBase { final BitSet parentDocs; final LeafBucketCollector sub; final DocIdSetIterator childDocs; - final LongArrayList bucketBuffer = new LongArrayList(); + final List bucketBuffer = new ArrayList<>(); Scorable scorer; int currentParentDoc = -1; @@ -221,10 +222,8 @@ void processBufferedChildBuckets() throws IOException { for (; childDocId < currentParentDoc; childDocId = childDocs.nextDoc()) { cachedScorer.doc = childDocId; - final long[] buffer = bucketBuffer.buffer; - final int size = bucketBuffer.size(); - for (int i = 0; i < size; i++) { - collectBucket(sub, childDocId, buffer[i]); + for (var bucket : bucketBuffer) { + collectBucket(sub, childDocId, bucket); } } bucketBuffer.clear(); diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java index 3e18ac6844ed2..8b0429d2379c6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java @@ -31,7 +31,6 @@ package org.opensearch.search.aggregations.bucket.nested; -import com.carrotsearch.hppc.LongIntHashMap; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; @@ -51,6 +50,7 @@ import org.opensearch.search.internal.SearchContext; import java.io.IOException; +import java.util.HashMap; import java.util.Map; /** @@ -91,7 +91,7 @@ protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final Leaf if (parentDocs == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } - final LongIntHashMap bucketOrdToLastCollectedParentDoc = new LongIntHashMap(32); + final Map bucketOrdToLastCollectedParentDoc = new HashMap<>(32); return new LeafBucketCollectorBase(sub, null) { @Override public void collect(int childDoc, long bucket) throws IOException { @@ -99,16 +99,15 @@ public void collect(int childDoc, long bucket) throws IOException { final int parentDoc = parentDocs.nextSetBit(childDoc); assert childDoc <= parentDoc && parentDoc != DocIdSetIterator.NO_MORE_DOCS; - int keySlot = bucketOrdToLastCollectedParentDoc.indexOf(bucket); - if (bucketOrdToLastCollectedParentDoc.indexExists(keySlot)) { - int lastCollectedParentDoc = bucketOrdToLastCollectedParentDoc.indexGet(keySlot); + Integer lastCollectedParentDoc = bucketOrdToLastCollectedParentDoc.get(bucket); + if (lastCollectedParentDoc != null) { if (parentDoc > lastCollectedParentDoc) { collectBucket(sub, parentDoc, bucket); - bucketOrdToLastCollectedParentDoc.indexReplace(keySlot, parentDoc); + bucketOrdToLastCollectedParentDoc.put(bucket, parentDoc); } } else { collectBucket(sub, parentDoc, bucket); - bucketOrdToLastCollectedParentDoc.indexInsert(keySlot, bucket, parentDoc); + bucketOrdToLastCollectedParentDoc.put(bucket, parentDoc); } } }; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java index 8f0e50f2348ae..f8061bcaca50f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/IncludeExclude.java @@ -31,9 +31,6 @@ package org.opensearch.search.aggregations.bucket.terms; -import com.carrotsearch.hppc.BitMixer; -import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongSet; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -48,6 +45,7 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.OpenSearchParseException; +import org.opensearch.common.util.BitMixer; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -200,15 +198,15 @@ public boolean accept(long value) { * @opensearch.internal */ public static class SetBackedLongFilter extends LongFilter { - private LongSet valids; - private LongSet invalids; + private Set valids; + private Set invalids; private SetBackedLongFilter(int numValids, int numInvalids) { if (numValids > 0) { - valids = new LongHashSet(numValids); + valids = new HashSet<>(numValids); } if (numInvalids > 0) { - invalids = new LongHashSet(numInvalids); + invalids = new HashSet<>(numInvalids); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java index 77d8c3cdcf46b..faae96b957b2d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java @@ -32,7 +32,6 @@ package org.opensearch.search.aggregations.metrics; -import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; @@ -44,6 +43,7 @@ import org.opensearch.common.hash.MurmurHash3; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitArray; +import org.opensearch.common.util.BitMixer; import org.opensearch.common.util.LongArray; import org.opensearch.common.util.ObjectArray; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java index 5242461fa0054..ee3cd2963c334 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java @@ -32,8 +32,6 @@ package org.opensearch.search.aggregations.metrics; -import com.carrotsearch.hppc.LongObjectHashMap; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FieldDoc; @@ -68,6 +66,7 @@ import org.opensearch.search.sort.SortAndFormats; import java.io.IOException; +import java.util.HashMap; import java.util.Map; /** @@ -129,7 +128,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol // when post collecting then we have already replaced the leaf readers on the aggregator level have already been // replaced with the next leaf readers and then post collection pushes docids of the previous segment, which // then causes assertions to trip or incorrect top docs to be computed. - final LongObjectHashMap leafCollectors = new LongObjectHashMap<>(1); + final Map leafCollectors = new HashMap<>(1); return new LeafBucketCollectorBase(sub, null) { Scorable scorer; @@ -138,8 +137,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol public void setScorer(Scorable scorer) throws IOException { this.scorer = scorer; super.setScorer(scorer); - for (ObjectCursor cursor : leafCollectors.values()) { - cursor.value.setScorer(scorer); + for (var collector : leafCollectors.values()) { + collector.setScorer(scorer); } } @@ -170,16 +169,13 @@ public void collect(int docId, long bucket) throws IOException { topDocsCollectors.put(bucket, collectors); } - final LeafCollector leafCollector; - final int key = leafCollectors.indexOf(bucket); - if (key < 0) { + LeafCollector leafCollector = leafCollectors.get(bucket); + if (leafCollector == null) { leafCollector = collectors.collector.getLeafCollector(ctx); if (scorer != null) { leafCollector.setScorer(scorer); } - leafCollectors.indexInsert(key, bucket, leafCollector); - } else { - leafCollector = leafCollectors.indexGet(key); + leafCollectors.put(bucket, leafCollector); } leafCollector.collect(docId); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java index 43dc8c6714663..f4ca3d18718ab 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java @@ -32,8 +32,6 @@ package org.opensearch.search.aggregations.pipeline; -import com.carrotsearch.hppc.DoubleArrayList; - import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -41,7 +39,9 @@ import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -175,11 +175,11 @@ protected PercentilesBucketPipelineAggregationBuilder buildFactory( protected boolean token(XContentParser parser, String field, XContentParser.Token token, Map params) throws IOException { if (PERCENTS_FIELD.match(field, parser.getDeprecationHandler()) && token == XContentParser.Token.START_ARRAY) { - DoubleArrayList percents = new DoubleArrayList(10); + final List percents = new ArrayList<>(10); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { percents.add(parser.doubleValue()); } - params.put(PERCENTS_FIELD.getPreferredName(), percents.toArray()); + params.put(PERCENTS_FIELD.getPreferredName(), percents.stream().mapToDouble(Double::doubleValue).toArray()); return true; } else if (KEYED_FIELD.match(field, parser.getDeprecationHandler()) && token == XContentParser.Token.VALUE_BOOLEAN) { params.put(KEYED_FIELD.getPreferredName(), parser.booleanValue()); diff --git a/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java b/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java index 0223699b7fd89..35ab0c0701b4e 100644 --- a/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java +++ b/server/src/main/java/org/opensearch/search/dfs/AggregatedDfs.java @@ -32,17 +32,17 @@ package org.opensearch.search.dfs; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; -import org.opensearch.common.collect.HppcMaps; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; /** * Compute global distributed frequency across the index @@ -51,25 +51,26 @@ */ public class AggregatedDfs implements Writeable { - private ObjectObjectHashMap termStatistics; - private ObjectObjectHashMap fieldStatistics; + private Map termStatistics; + private Map fieldStatistics; private long maxDoc; public AggregatedDfs(StreamInput in) throws IOException { int size = in.readVInt(); - termStatistics = HppcMaps.newMap(size); + final Map termStatistics = new HashMap<>(size); for (int i = 0; i < size; i++) { Term term = new Term(in.readString(), in.readBytesRef()); TermStatistics stats = new TermStatistics(in.readBytesRef(), in.readVLong(), DfsSearchResult.subOne(in.readVLong())); termStatistics.put(term, stats); } - fieldStatistics = DfsSearchResult.readFieldStats(in); + this.termStatistics = Collections.unmodifiableMap(termStatistics); + this.fieldStatistics = DfsSearchResult.readFieldStats(in); maxDoc = in.readVLong(); } public AggregatedDfs( - ObjectObjectHashMap termStatistics, - ObjectObjectHashMap fieldStatistics, + final Map termStatistics, + final Map fieldStatistics, long maxDoc ) { this.termStatistics = termStatistics; @@ -77,11 +78,11 @@ public AggregatedDfs( this.maxDoc = maxDoc; } - public ObjectObjectHashMap termStatistics() { + public Map termStatistics() { return termStatistics; } - public ObjectObjectHashMap fieldStatistics() { + public Map fieldStatistics() { return fieldStatistics; } @@ -93,11 +94,11 @@ public long maxDoc() { public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(termStatistics.size()); - for (ObjectObjectCursor c : termStatistics()) { - Term term = c.key; + for (final Map.Entry c : termStatistics().entrySet()) { + Term term = c.getKey(); out.writeString(term.field()); out.writeBytesRef(term.bytes()); - TermStatistics stats = c.value; + TermStatistics stats = c.getValue(); out.writeBytesRef(stats.term()); out.writeVLong(stats.docFreq()); out.writeVLong(DfsSearchResult.addOne(stats.totalTermFreq())); diff --git a/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java b/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java index 6814fbd6e32c5..5094145ea1c6e 100644 --- a/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java @@ -32,14 +32,12 @@ package org.opensearch.search.dfs; -import com.carrotsearch.hppc.ObjectObjectHashMap; import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TermStatistics; -import org.opensearch.common.collect.HppcMaps; import org.opensearch.search.internal.SearchContext; import org.opensearch.search.rescore.RescoreContext; import org.opensearch.tasks.TaskCancelledException; @@ -58,7 +56,7 @@ public class DfsPhase { public void execute(SearchContext context) { try { - ObjectObjectHashMap fieldStatistics = HppcMaps.newNoNullKeysMap(); + Map fieldStatistics = new HashMap<>(); Map stats = new HashMap<>(); IndexSearcher searcher = new IndexSearcher(context.searcher().getIndexReader()) { @Override diff --git a/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java index b46dff0dbe8ef..0209f91f7fa52 100644 --- a/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java +++ b/server/src/main/java/org/opensearch/search/dfs/DfsSearchResult.java @@ -32,13 +32,10 @@ package org.opensearch.search.dfs; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.collect.HppcMaps; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchPhaseResult; @@ -47,6 +44,9 @@ import org.opensearch.search.internal.ShardSearchRequest; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; /** * Result from a Distributed Frequency Search phase @@ -59,7 +59,7 @@ public class DfsSearchResult extends SearchPhaseResult { private static final TermStatistics[] EMPTY_TERM_STATS = new TermStatistics[0]; private Term[] terms; private TermStatistics[] termStatistics; - private ObjectObjectHashMap fieldStatistics = HppcMaps.newNoNullKeysMap(); + private Map fieldStatistics; private int maxDoc; public DfsSearchResult(StreamInput in) throws IOException { @@ -75,7 +75,7 @@ public DfsSearchResult(StreamInput in) throws IOException { } } this.termStatistics = readTermStats(in, terms); - fieldStatistics = readFieldStats(in); + this.fieldStatistics = readFieldStats(in); maxDoc = in.readVInt(); setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new)); @@ -102,8 +102,8 @@ public DfsSearchResult termsStatistics(Term[] terms, TermStatistics[] termStatis return this; } - public DfsSearchResult fieldStatistics(ObjectObjectHashMap fieldStatistics) { - this.fieldStatistics = fieldStatistics; + public DfsSearchResult fieldStatistics(final Map fieldStatistics) { + this.fieldStatistics = Collections.unmodifiableMap(fieldStatistics); return this; } @@ -115,7 +115,7 @@ public TermStatistics[] termStatistics() { return termStatistics; } - public ObjectObjectHashMap fieldStatistics() { + public Map fieldStatistics() { return fieldStatistics; } @@ -133,13 +133,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(getShardSearchRequest()); } - public static void writeFieldStats(StreamOutput out, ObjectObjectHashMap fieldStatistics) - throws IOException { + public static void writeFieldStats(StreamOutput out, final Map fieldStatistics) throws IOException { out.writeVInt(fieldStatistics.size()); - for (ObjectObjectCursor c : fieldStatistics) { - out.writeString(c.key); - CollectionStatistics statistics = c.value; + for (final Map.Entry c : fieldStatistics.entrySet()) { + out.writeString(c.getKey()); + CollectionStatistics statistics = c.getValue(); assert statistics.maxDoc() >= 0; out.writeVLong(statistics.maxDoc()); // stats are always positive numbers @@ -167,9 +166,9 @@ public static void writeSingleTermStats(StreamOutput out, TermStatistics termSta } } - static ObjectObjectHashMap readFieldStats(StreamInput in) throws IOException { + static Map readFieldStats(StreamInput in) throws IOException { final int numFieldStatistics = in.readVInt(); - ObjectObjectHashMap fieldStatistics = HppcMaps.newNoNullKeysMap(numFieldStatistics); + final Map fieldStatistics = new HashMap<>(numFieldStatistics); for (int i = 0; i < numFieldStatistics; i++) { final String field = in.readString(); assert field != null; @@ -184,7 +183,7 @@ static ObjectObjectHashMap readFieldStats(StreamIn CollectionStatistics stats = new CollectionStatistics(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq); fieldStatistics.put(field, stats); } - return fieldStatistics; + return Collections.unmodifiableMap(fieldStatistics); } static TermStatistics[] readTermStats(StreamInput in, Term[] terms) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java b/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java index 8c009b9f5f672..85d003db5726f 100644 --- a/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java +++ b/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java @@ -32,7 +32,6 @@ package org.opensearch.search.slice; -import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedNumericDocValues; @@ -44,6 +43,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; +import org.opensearch.common.util.BitMixer; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 6edf78b54aa70..54d55f67ccdcd 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -31,8 +31,6 @@ package org.opensearch.snapshots; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -499,7 +497,7 @@ public ClusterState execute(ClusterState currentState) { } // Check that the index is closed or doesn't exist IndexMetadata currentIndexMetadata = currentState.metadata().index(renamedIndexName); - IntSet ignoreShards = new IntHashSet(); + Set ignoreShards = new HashSet<>(); final Index renamedIndex; if (currentIndexMetadata == null) { // Index doesn't exist - create it and start recovery @@ -692,7 +690,7 @@ private void checkAliasNameConflicts(Map renamedIndices, Set ignoreShards) { for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) { if (index.equals(failure.index())) { ignoreShards.add(failure.shardId()); diff --git a/server/src/main/java/org/opensearch/tasks/TaskManager.java b/server/src/main/java/org/opensearch/tasks/TaskManager.java index f243cf392bbb8..bcd3004188bb6 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskManager.java +++ b/server/src/main/java/org/opensearch/tasks/TaskManager.java @@ -32,8 +32,6 @@ package org.opensearch.tasks; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.ObjectIntMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -80,7 +78,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import static org.opensearch.common.unit.TimeValue.timeValueMillis; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE; @@ -584,7 +581,7 @@ private static class CancellableTaskHolder { private final CancellableTask task; private boolean finished = false; private List cancellationListeners = null; - private ObjectIntMap childTasksPerNode = null; + private Map childTasksPerNode = null; private boolean banChildren = false; private List childTaskCompletedListeners = null; @@ -666,15 +663,15 @@ synchronized void registerChildNode(DiscoveryNode node) { throw new TaskCancelledException("The parent task was cancelled, shouldn't start any child tasks"); } if (childTasksPerNode == null) { - childTasksPerNode = new ObjectIntHashMap<>(); + childTasksPerNode = new HashMap<>(); } - childTasksPerNode.addTo(node, 1); + childTasksPerNode.merge(node, 1, Integer::sum); } void unregisterChildNode(DiscoveryNode node) { final List listeners; synchronized (this) { - if (childTasksPerNode.addTo(node, -1) == 0) { + if (childTasksPerNode.merge(node, -1, Integer::sum) == 0) { childTasksPerNode.remove(node); } if (childTasksPerNode.isEmpty() && this.childTaskCompletedListeners != null) { @@ -695,9 +692,7 @@ Set startBan(Runnable onChildTasksCompleted) { if (childTasksPerNode == null) { pendingChildNodes = Collections.emptySet(); } else { - pendingChildNodes = StreamSupport.stream(childTasksPerNode.spliterator(), false) - .map(e -> e.key) - .collect(Collectors.toSet()); + pendingChildNodes = Set.copyOf(childTasksPerNode.keySet()); } if (pendingChildNodes.isEmpty()) { assert childTaskCompletedListeners == null; diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 0ea383d6d48d7..75676307f4026 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -31,8 +31,6 @@ package org.opensearch.transport; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -524,12 +522,12 @@ static int resolvePublishPort(ProfileSettings profileSettings, List ports = new HashSet<>(); for (InetSocketAddress boundAddress : boundAddresses) { ports.add(boundAddress.getPort()); } if (ports.size() == 1) { - publishPort = ports.iterator().next().value; + publishPort = ports.iterator().next(); } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java index d29c9cb30a09c..e1838f7605816 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java @@ -37,7 +37,6 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.UUIDs; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -59,10 +58,10 @@ public class IndicesShardStoreResponseTests extends OpenSearchTestCase { public void testBasicSerialization() throws Exception { - final Map>> indexStoreStatuses = new HashMap<>(); + final Map>> indexStoreStatuses = new HashMap<>(); List failures = new ArrayList<>(); - ImmutableOpenIntMap.Builder> storeStatuses = ImmutableOpenIntMap.builder(); + final Map> storeStatuses = new HashMap<>(); DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -88,7 +87,7 @@ public void testBasicSerialization() throws Exception { ); storeStatuses.put(0, storeStatusList); storeStatuses.put(1, storeStatusList); - ImmutableOpenIntMap> storesMap = storeStatuses.build(); + final Map> storesMap = Collections.unmodifiableMap(storeStatuses); indexStoreStatuses.put("test", storesMap); indexStoreStatuses.put("test2", storesMap); diff --git a/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java b/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java index 302f3c1efc394..991d118e5243b 100644 --- a/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java +++ b/server/src/test/java/org/opensearch/action/support/ActiveShardCountTests.java @@ -32,7 +32,6 @@ package org.opensearch.action.support; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -231,8 +230,8 @@ private ClusterState startPrimaries(final ClusterState clusterState, final Strin RoutingTable routingTable = clusterState.routingTable(); IndexRoutingTable indexRoutingTable = routingTable.index(indexName); IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary()) { shardRouting = shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize()) @@ -249,8 +248,8 @@ private ClusterState startLessThanWaitOnShards(final ClusterState clusterState, RoutingTable routingTable = clusterState.routingTable(); IndexRoutingTable indexRoutingTable = routingTable.index(indexName); IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; assert shardRoutingTable.getSize() > 2; int numToStart = numShardsToStart; // want less than half, and primary is already started @@ -275,8 +274,8 @@ private ClusterState startWaitOnShards(final ClusterState clusterState, final St RoutingTable routingTable = clusterState.routingTable(); IndexRoutingTable indexRoutingTable = routingTable.index(indexName); IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; assert shardRoutingTable.getSize() > 2; int numToStart = numShardsToStart; for (ShardRouting shardRouting : shardRoutingTable.getShards()) { @@ -304,8 +303,8 @@ private ClusterState startAllShards(final ClusterState clusterState, final Strin RoutingTable routingTable = clusterState.routingTable(); IndexRoutingTable indexRoutingTable = routingTable.index(indexName); IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary()) { assertTrue(shardRouting.active()); diff --git a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java index fde0ebe4b4e30..0e8af8bb0a9a8 100644 --- a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java @@ -31,8 +31,6 @@ package org.opensearch.cluster.health; -import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.Version; import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; @@ -56,7 +54,6 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; -import org.opensearch.common.collect.ImmutableOpenIntMap; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; @@ -76,8 +73,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -416,8 +415,8 @@ private List generateClusterStates( RoutingTable routingTable = originalClusterState.routingTable(); IndexRoutingTable indexRoutingTable = routingTable.index(indexName); IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (final ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary()) { newIndexRoutingTable.addShard(shardRouting.initialize(randomFrom(nodeIds), null, shardRouting.getExpectedShardSize())); @@ -433,13 +432,13 @@ private List generateClusterStates( // some primaries started indexRoutingTable = routingTable.index(indexName); newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - ImmutableOpenIntMap.Builder> allocationIds = ImmutableOpenIntMap.>builder(); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + Map> allocationIds = new HashMap<>(); + for (var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (final ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary() && randomBoolean()) { final ShardRouting newShardRouting = shardRouting.moveToStarted(); - allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId())); + allocationIds.put(newShardRouting.getId(), Set.of(newShardRouting.allocationId().getId())); newIndexRoutingTable.addShard(newShardRouting); } else { newIndexRoutingTable.addShard(shardRouting); @@ -447,10 +446,8 @@ private List generateClusterStates( } } routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build(); - IndexMetadata.Builder idxMetaBuilder = IndexMetadata.builder(clusterState.metadata().index(indexName)); - for (final IntObjectCursor> entry : allocationIds.build()) { - idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value); - } + final IndexMetadata.Builder idxMetaBuilder = IndexMetadata.builder(clusterState.metadata().index(indexName)); + allocationIds.forEach(idxMetaBuilder::putInSyncAllocationIds); Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(idxMetaBuilder); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metadata(metadataBuilder).build(); clusterStates.add(clusterState); @@ -460,8 +457,8 @@ private List generateClusterStates( // some primaries failed to allocate indexRoutingTable = routingTable.index(indexName); newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (final ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary() && (shardRouting.started() == false || alreadyFailedPrimary == false)) { newIndexRoutingTable.addShard( @@ -481,13 +478,13 @@ private List generateClusterStates( // all primaries started indexRoutingTable = routingTable.index(indexName); newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - allocationIds = ImmutableOpenIntMap.>builder(); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + allocationIds = new HashMap<>(); + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (final ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary() && shardRouting.started() == false) { final ShardRouting newShardRouting = shardRouting.moveToStarted(); - allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId())); + allocationIds.put(newShardRouting.getId(), Set.of(newShardRouting.allocationId().getId())); newIndexRoutingTable.addShard(newShardRouting); } else { newIndexRoutingTable.addShard(shardRouting); @@ -495,19 +492,17 @@ private List generateClusterStates( } } routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build(); - idxMetaBuilder = IndexMetadata.builder(clusterState.metadata().index(indexName)); - for (final IntObjectCursor> entry : allocationIds.build()) { - idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value); - } - metadataBuilder = Metadata.builder(clusterState.metadata()).put(idxMetaBuilder); + final IndexMetadata.Builder idxMetaBuilder2 = IndexMetadata.builder(clusterState.metadata().index(indexName)); + allocationIds.forEach(idxMetaBuilder2::putInSyncAllocationIds); + metadataBuilder = Metadata.builder(clusterState.metadata()).put(idxMetaBuilder2); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metadata(metadataBuilder).build(); clusterStates.add(clusterState); // initialize replicas indexRoutingTable = routingTable.index(indexName); newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId(); Set allocatedNodes = new HashSet<>(); allocatedNodes.add(primaryNodeId); @@ -528,8 +523,8 @@ private List generateClusterStates( // some replicas started indexRoutingTable = routingTable.index(indexName); newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (final ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary() == false && randomBoolean()) { newIndexRoutingTable.addShard(shardRouting.moveToStarted()); @@ -545,8 +540,8 @@ private List generateClusterStates( boolean replicaStateChanged = false; indexRoutingTable = routingTable.index(indexName); newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (final ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary() == false && shardRouting.started() == false) { newIndexRoutingTable.addShard(shardRouting.moveToStarted()); @@ -568,10 +563,10 @@ private List generateClusterStates( // returns true if the inactive primaries in the index are only due to cluster recovery // (not because of allocation of existing shard or previously having allocation ids assigned) private boolean primaryInactiveDueToRecovery(final String indexName, final ClusterState clusterState) { - for (final IntObjectCursor shardRouting : clusterState.routingTable().index(indexName).shards()) { - final ShardRouting primaryShard = shardRouting.value.primaryShard(); + for (final var shardRouting : clusterState.routingTable().index(indexName).shards().entrySet()) { + final ShardRouting primaryShard = shardRouting.getValue().primaryShard(); if (primaryShard.active() == false) { - if (clusterState.metadata().index(indexName).inSyncAllocationIds(shardRouting.key).isEmpty() == false) { + if (clusterState.metadata().index(indexName).inSyncAllocationIds(shardRouting.getKey()).isEmpty() == false) { return false; } if (primaryShard.recoverySource() != null diff --git a/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java b/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java index 28bf99deec6f4..741bf8e34c0e3 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/UnassignedInfoTests.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.routing; -import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.opensearch.Version; import org.opensearch.cluster.ClusterName; @@ -58,6 +57,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -202,7 +202,7 @@ public void testNewIndexRestored() { Version.CURRENT, new IndexId("test", UUIDs.randomBase64UUID(random())) ), - new IntHashSet() + new HashSet<>() ) .build() ) diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsAllocateUnassignedTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsAllocateUnassignedTests.java index 7c45b20ecee1f..ed178ed7e1526 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsAllocateUnassignedTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsAllocateUnassignedTests.java @@ -8,7 +8,6 @@ package org.opensearch.cluster.routing.allocation; -import com.carrotsearch.hppc.ObjectIntHashMap; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.RoutingNode; import org.opensearch.cluster.routing.RoutingNodes; @@ -16,6 +15,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.allocator.RemoteShardsBalancer; +import java.util.HashMap; import java.util.Map; public class RemoteShardsAllocateUnassignedTests extends RemoteShardsBalancerBaseTestCase { @@ -68,7 +68,7 @@ public void testPrimaryAllocation() { assertEquals(0, routingNodes.unassigned().size()); - ObjectIntHashMap nodePrimariesCounter = new ObjectIntHashMap<>(); + final Map nodePrimariesCounter = new HashMap<>(); for (ShardRouting shard : clusterState.getRoutingTable().allShards()) { assertFalse(shard.unassigned()); RoutingNode node = routingNodes.node(shard.currentNodeId()); @@ -80,11 +80,11 @@ public void testPrimaryAllocation() { if (RoutingPool.getNodePool(node) == RoutingPool.REMOTE_CAPABLE && RoutingPool.REMOTE_CAPABLE.equals(RoutingPool.getShardPool(shard, allocation)) && shard.primary()) { - nodePrimariesCounter.putOrAdd(node.nodeId(), 1, 1); + nodePrimariesCounter.compute(node.nodeId(), (k, v) -> (v == null) ? 1 : v + 1); } } final int indexShardLimit = (int) Math.ceil(totalPrimaries(remoteIndices) / (float) remoteCapableNodes); - for (int primaries : nodePrimariesCounter.values) { + for (int primaries : nodePrimariesCounter.values()) { assertTrue(primaries <= indexShardLimit); } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java index c4358aaf12ac2..ef9ae90e18bb5 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java @@ -8,13 +8,15 @@ package org.opensearch.cluster.routing.allocation; -import com.carrotsearch.hppc.ObjectIntHashMap; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.RoutingNode; import org.opensearch.cluster.routing.RoutingNodes; import org.opensearch.cluster.routing.RoutingPool; import org.opensearch.cluster.routing.ShardRouting; +import java.util.HashMap; +import java.util.Map; + public class RemoteShardsRebalanceShardsTests extends RemoteShardsBalancerBaseTestCase { /** @@ -33,8 +35,8 @@ public void testShardAllocationAndRebalance() { RoutingNodes routingNodes = clusterState.getRoutingNodes(); RoutingAllocation allocation = getRoutingAllocation(clusterState, routingNodes); - ObjectIntHashMap nodePrimariesCounter = getShardCounterPerNodeForRemoteCapablePool(clusterState, allocation, true); - ObjectIntHashMap nodeReplicaCounter = getShardCounterPerNodeForRemoteCapablePool(clusterState, allocation, false); + final Map nodePrimariesCounter = getShardCounterPerNodeForRemoteCapablePool(clusterState, allocation, true); + final Map nodeReplicaCounter = getShardCounterPerNodeForRemoteCapablePool(clusterState, allocation, false); int avgPrimariesPerNode = getTotalShardCountAcrossNodes(nodePrimariesCounter) / remoteCapableNodes; // Primary and replica are balanced post first reroute @@ -46,23 +48,23 @@ public void testShardAllocationAndRebalance() { } } - private ObjectIntHashMap getShardCounterPerNodeForRemoteCapablePool( + private Map getShardCounterPerNodeForRemoteCapablePool( ClusterState clusterState, RoutingAllocation allocation, boolean primary ) { - ObjectIntHashMap nodePrimariesCounter = new ObjectIntHashMap<>(); + final Map nodePrimariesCounter = new HashMap<>(); for (ShardRouting shard : clusterState.getRoutingTable().allShards()) { if (RoutingPool.REMOTE_CAPABLE.equals(RoutingPool.getShardPool(shard, allocation)) && shard.primary() == primary) { - nodePrimariesCounter.putOrAdd(shard.currentNodeId(), 1, 1); + nodePrimariesCounter.compute(shard.currentNodeId(), (k, v) -> (v == null) ? 1 : v + 1); } } return nodePrimariesCounter; } - private int getTotalShardCountAcrossNodes(ObjectIntHashMap nodePrimariesCounter) { + private int getTotalShardCountAcrossNodes(final Map nodePrimariesCounter) { int totalShardCount = 0; - for (int value : nodePrimariesCounter.values) { + for (int value : nodePrimariesCounter.values()) { totalShardCount += value; } return totalShardCount; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java index e4bbe3c71d918..1ccadcade26bb 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.routing.allocation; -import com.carrotsearch.hppc.IntHashSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; @@ -804,7 +803,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations( Version.CURRENT, new IndexId(indexMetadata.getIndex().getName(), UUIDs.randomBase64UUID(random())) ), - new IntHashSet() + new HashSet<>() ); break; case 4: diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index de155f3fdb896..2180a14f5bf30 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.routing.allocation.decider; -import com.carrotsearch.hppc.IntHashSet; import org.opensearch.Version; import org.opensearch.cluster.ClusterInfo; import org.opensearch.cluster.ClusterInfoService; @@ -1526,7 +1525,7 @@ public void testDiskThresholdWithSnapshotShardSizes() { .addAsNewRestore( metadata.index("test"), new RecoverySource.SnapshotRecoverySource("_restore_uuid", snapshot, Version.CURRENT, indexId), - new IntHashSet() + new HashSet<>() ) .build(); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java index 8cca157210c37..7e2f258cd0021 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java @@ -31,7 +31,6 @@ package org.opensearch.cluster.routing.allocation.decider; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -145,8 +144,8 @@ public void testCanAllocatePrimaryExistingInRestoreInProgress() { IndexRoutingTable indexRoutingTable = routingTable.index("test"); IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex()); - for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) { - final IndexShardRoutingTable shardRoutingTable = shardEntry.value; + for (final var shardEntry : indexRoutingTable.getShards().values()) { + final IndexShardRoutingTable shardRoutingTable = shardEntry; for (ShardRouting shardRouting : shardRoutingTable.getShards()) { if (shardRouting.primary()) { newIndexRoutingTable.addShard(primary); diff --git a/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java b/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java index 2d895f0a10412..e2c155dd47d80 100644 --- a/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java +++ b/server/src/test/java/org/opensearch/common/FieldMemoryStatsTests.java @@ -31,12 +31,13 @@ package org.opensearch.common; -import com.carrotsearch.hppc.ObjectLongHashMap; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; public class FieldMemoryStatsTests extends OpenSearchTestCase { @@ -54,21 +55,21 @@ public void testHashCodeEquals() { FieldMemoryStats stats = randomFieldMemoryStats(); assertEquals(stats, stats); assertEquals(stats.hashCode(), stats.hashCode()); - ObjectLongHashMap map1 = new ObjectLongHashMap<>(); - map1.put("bar", 1); + final Map map1 = new HashMap<>(); + map1.put("bar", 1L); FieldMemoryStats stats1 = new FieldMemoryStats(map1); - ObjectLongHashMap map2 = new ObjectLongHashMap<>(); - map2.put("foo", 2); + final Map map2 = new HashMap<>(); + map2.put("foo", 2L); FieldMemoryStats stats2 = new FieldMemoryStats(map2); - ObjectLongHashMap map3 = new ObjectLongHashMap<>(); - map3.put("foo", 2); - map3.put("bar", 1); + final Map map3 = new HashMap<>(); + map3.put("foo", 2L); + map3.put("bar", 1L); FieldMemoryStats stats3 = new FieldMemoryStats(map3); - ObjectLongHashMap map4 = new ObjectLongHashMap<>(); - map4.put("foo", 2); - map4.put("bar", 1); + final Map map4 = new HashMap<>(); + map4.put("foo", 2L); + map4.put("bar", 1L); FieldMemoryStats stats4 = new FieldMemoryStats(map4); assertNotEquals(stats1, stats2); @@ -83,21 +84,21 @@ public void testHashCodeEquals() { } public void testAdd() { - ObjectLongHashMap map1 = new ObjectLongHashMap<>(); - map1.put("bar", 1); + final Map map1 = new HashMap<>(); + map1.put("bar", 1L); FieldMemoryStats stats1 = new FieldMemoryStats(map1); - ObjectLongHashMap map2 = new ObjectLongHashMap<>(); - map2.put("foo", 2); + final Map map2 = new HashMap<>(); + map2.put("foo", 2L); FieldMemoryStats stats2 = new FieldMemoryStats(map2); - ObjectLongHashMap map3 = new ObjectLongHashMap<>(); - map3.put("bar", 1); + final Map map3 = new HashMap<>(); + map3.put("bar", 1L); FieldMemoryStats stats3 = new FieldMemoryStats(map3); stats3.add(stats1); - ObjectLongHashMap map4 = new ObjectLongHashMap<>(); - map4.put("foo", 2); - map4.put("bar", 2); + final Map map4 = new HashMap<>(); + map4.put("foo", 2L); + map4.put("bar", 2L); FieldMemoryStats stats4 = new FieldMemoryStats(map4); assertNotEquals(stats3, stats4); stats3.add(stats2); @@ -105,7 +106,7 @@ public void testAdd() { } public static FieldMemoryStats randomFieldMemoryStats() { - ObjectLongHashMap map = new ObjectLongHashMap<>(); + final Map map = new HashMap<>(); int keys = randomIntBetween(1, 1000); for (int i = 0; i < keys; i++) { map.put(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomNonNegativeLong()); diff --git a/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java b/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java deleted file mode 100644 index 42450bb69ce75..0000000000000 --- a/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.hppc; - -import com.carrotsearch.hppc.ObjectHashSet; -import org.opensearch.core.Assertions; -import org.opensearch.common.collect.HppcMaps; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; - -public class HppcMapsTests extends OpenSearchTestCase { - public void testIntersection() throws Exception { - assumeTrue("assertions enabled", Assertions.ENABLED); - ObjectHashSet set1 = ObjectHashSet.from("1", "2", "3"); - ObjectHashSet set2 = ObjectHashSet.from("1", "2", "3"); - List values = toList(HppcMaps.intersection(set1, set2)); - assertThat(values.size(), equalTo(3)); - assertThat(values.contains("1"), equalTo(true)); - assertThat(values.contains("2"), equalTo(true)); - assertThat(values.contains("3"), equalTo(true)); - - set1 = ObjectHashSet.from("1", "2", "3"); - set2 = ObjectHashSet.from("3", "4", "5"); - values = toList(HppcMaps.intersection(set1, set2)); - assertThat(values.size(), equalTo(1)); - assertThat(values.get(0), equalTo("3")); - - set1 = ObjectHashSet.from("1", "2", "3"); - set2 = ObjectHashSet.from("4", "5", "6"); - values = toList(HppcMaps.intersection(set1, set2)); - assertThat(values.size(), equalTo(0)); - - set1 = ObjectHashSet.from(); - set2 = ObjectHashSet.from("3", "4", "5"); - values = toList(HppcMaps.intersection(set1, set2)); - assertThat(values.size(), equalTo(0)); - - set1 = ObjectHashSet.from("1", "2", "3"); - set2 = ObjectHashSet.from(); - values = toList(HppcMaps.intersection(set1, set2)); - assertThat(values.size(), equalTo(0)); - - set1 = ObjectHashSet.from(); - set2 = ObjectHashSet.from(); - values = toList(HppcMaps.intersection(set1, set2)); - assertThat(values.size(), equalTo(0)); - - set1 = null; - set2 = ObjectHashSet.from(); - try { - toList(HppcMaps.intersection(set1, set2)); - fail(); - } catch (AssertionError e) {} - - set1 = ObjectHashSet.from(); - set2 = null; - try { - toList(HppcMaps.intersection(set1, set2)); - fail(); - } catch (AssertionError e) {} - - set1 = null; - set2 = null; - try { - toList(HppcMaps.intersection(set1, set2)); - fail(); - } catch (AssertionError e) {} - } - - private List toList(Iterable iterable) { - List list = new ArrayList<>(); - for (String s : iterable) { - list.add(s); - } - return list; - } - -} diff --git a/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java b/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java index 1859a7d4a8f9d..8b719283ed71d 100644 --- a/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java +++ b/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java @@ -32,9 +32,6 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.ObjectLongHashMap; -import com.carrotsearch.hppc.ObjectLongMap; -import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.tests.util.TestUtil; @@ -77,7 +74,7 @@ public void testDuel() { for (int i = 0; i < values.length; ++i) { values[i] = new BytesRef(randomAlphaOfLength(5)); } - final ObjectLongMap valueToId = new ObjectLongHashMap<>(); + final Map valueToId = new HashMap<>(); final BytesRef[] idToValue = new BytesRef[values.length]; final int iters = randomInt(1000000); for (int i = 0; i < iters; ++i) { @@ -92,8 +89,8 @@ public void testDuel() { } assertEquals(valueToId.size(), hash.size()); - for (final ObjectLongCursor next : valueToId) { - assertEquals(next.value, hash.find(next.key, next.key.hashCode())); + for (final var next : valueToId.entrySet()) { + assertEquals(next.getValue().longValue(), hash.find(next.getKey(), next.getKey().hashCode())); } for (long i = 0; i < hash.capacity(); ++i) { diff --git a/server/src/test/java/org/opensearch/common/util/LongHashTests.java b/server/src/test/java/org/opensearch/common/util/LongHashTests.java index aabc3c295e6fa..295497b9f188c 100644 --- a/server/src/test/java/org/opensearch/common/util/LongHashTests.java +++ b/server/src/test/java/org/opensearch/common/util/LongHashTests.java @@ -32,17 +32,12 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.LongLongHashMap; -import com.carrotsearch.hppc.LongLongMap; -import com.carrotsearch.hppc.cursors.LongLongCursor; - import org.opensearch.common.settings.Settings; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -74,7 +69,7 @@ public void testDuell() { for (int i = 0; i < values.length; ++i) { values[i] = randomLong(); } - final LongLongMap valueToId = new LongLongHashMap(); + final Map valueToId = new HashMap<>(); final long[] idToValue = new long[values.length]; final int iters = randomInt(1000000); for (int i = 0; i < iters; ++i) { @@ -89,9 +84,8 @@ public void testDuell() { } assertEquals(valueToId.size(), hash.size()); - for (Iterator iterator = valueToId.iterator(); iterator.hasNext();) { - final LongLongCursor next = iterator.next(); - assertEquals(next.value, hash.find(next.key)); + for (var iterator : valueToId.entrySet()) { + assertEquals(iterator.getValue().longValue(), hash.find(iterator.getKey())); } for (long i = 0; i < hash.capacity(); ++i) { diff --git a/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java b/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java index f99f1f8a9a1d5..73a6fc5d54883 100644 --- a/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java +++ b/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java @@ -32,11 +32,13 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.LongObjectHashMap; import org.opensearch.common.settings.Settings; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; +import java.util.HashMap; +import java.util.Map; + public class LongObjectHashMapTests extends OpenSearchTestCase { private BigArrays randomBigArrays() { @@ -44,7 +46,7 @@ private BigArrays randomBigArrays() { } public void testDuel() { - final LongObjectHashMap map1 = new LongObjectHashMap<>(); + final Map map1 = new HashMap<>(); final LongObjectPagedHashMap map2 = new LongObjectPagedHashMap<>( randomInt(42), 0.6f + randomFloat() * 0.39f, @@ -66,10 +68,10 @@ public void testDuel() { assertEquals(map1.size(), map2.size()); } } - for (int i = 0; i <= maxKey; ++i) { + for (long i = 0; i <= maxKey; ++i) { assertSame(map1.get(i), map2.get(i)); } - final LongObjectHashMap copy = new LongObjectHashMap<>(); + final Map copy = new HashMap<>(); for (LongObjectPagedHashMap.Cursor cursor : map2) { copy.put(cursor.key, cursor.value); } diff --git a/server/src/test/java/org/opensearch/index/engine/CombinedDeletionPolicyTests.java b/server/src/test/java/org/opensearch/index/engine/CombinedDeletionPolicyTests.java index dac6fa90bd917..dd469644709ec 100644 --- a/server/src/test/java/org/opensearch/index/engine/CombinedDeletionPolicyTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CombinedDeletionPolicyTests.java @@ -32,7 +32,6 @@ package org.opensearch.index.engine; -import com.carrotsearch.hppc.LongArrayList; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.store.Directory; import org.opensearch.index.seqno.RetentionLeases; @@ -74,7 +73,7 @@ public void testKeepCommitsAfterGlobalCheckpoint() throws Exception { TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy(); CombinedDeletionPolicy indexPolicy = newCombinedDeletionPolicy(translogPolicy, softDeletesPolicy, globalCheckpoint); - final LongArrayList maxSeqNoList = new LongArrayList(); + final List maxSeqNoList = new ArrayList<>(); final List commitList = new ArrayList<>(); int totalCommits = between(2, 20); long lastMaxSeqNo = 0; diff --git a/server/src/test/java/org/opensearch/index/seqno/LocalCheckpointTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/LocalCheckpointTrackerTests.java index 7a79c58768575..d79b7acfe643b 100644 --- a/server/src/test/java/org/opensearch/index/seqno/LocalCheckpointTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/LocalCheckpointTrackerTests.java @@ -172,7 +172,7 @@ public void testSimpleOverFlow() { assertThat(tracker.processedCheckpoint.get(), equalTo(maxOps - 1L)); assertThat(tracker.processedSeqNo.size(), equalTo(aligned ? 0 : 1)); if (aligned == false) { - assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE)); + assertThat(tracker.processedSeqNo.keySet().iterator().next(), equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE)); } assertThat(tracker.hasProcessed(randomFrom(seqNoList)), equalTo(true)); final long notCompletedSeqNo = randomValueOtherThanMany(seqNoList::contains, OpenSearchTestCase::randomNonNegativeLong); @@ -218,7 +218,7 @@ protected void doRun() throws Exception { assertThat(tracker.getProcessedCheckpoint(), equalTo(maxOps - 1L)); assertThat(tracker.processedSeqNo.size(), is(oneOf(0, 1))); if (tracker.processedSeqNo.size() == 1) { - assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE)); + assertThat(tracker.processedSeqNo.keySet().iterator().next(), equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE)); } } @@ -272,7 +272,7 @@ protected void doRun() throws Exception { assertThat(tracker.hasProcessed(randomLongBetween(maxOps, Long.MAX_VALUE)), equalTo(false)); assertThat(tracker.processedSeqNo.size(), is(oneOf(0, 1))); if (tracker.processedSeqNo.size() == 1) { - assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE)); + assertThat(tracker.processedSeqNo.keySet().iterator().next(), equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE)); } } diff --git a/server/src/test/java/org/opensearch/index/translog/MultiSnapshotTests.java b/server/src/test/java/org/opensearch/index/translog/MultiSnapshotTests.java index 783c42f1749c0..b5bef0c028665 100644 --- a/server/src/test/java/org/opensearch/index/translog/MultiSnapshotTests.java +++ b/server/src/test/java/org/opensearch/index/translog/MultiSnapshotTests.java @@ -32,12 +32,12 @@ package org.opensearch.index.translog; -import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongSet; import org.opensearch.common.Randomness; import org.opensearch.test.OpenSearchTestCase; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; @@ -60,7 +60,7 @@ public void testTrackSeqNoSimpleRange() throws Exception { public void testTrackSeqNoDenseRanges() throws Exception { final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); - final LongSet normalSet = new LongHashSet(); + final Set normalSet = new HashSet<>(); IntStream.range(0, scaledRandomIntBetween(5_000, 10_000)).forEach(i -> { long seq = between(0, 5000); boolean existed = normalSet.add(seq) == false; @@ -70,7 +70,7 @@ public void testTrackSeqNoDenseRanges() throws Exception { public void testTrackSeqNoSparseRanges() throws Exception { final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); - final LongSet normalSet = new LongHashSet(); + final Set normalSet = new HashSet<>(); IntStream.range(0, scaledRandomIntBetween(5_000, 10_000)).forEach(i -> { long seq = between(i * 10_000, i * 30_000); boolean existed = normalSet.add(seq) == false; @@ -80,7 +80,7 @@ public void testTrackSeqNoSparseRanges() throws Exception { public void testTrackSeqNoMimicTranslogRanges() throws Exception { final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); - final LongSet normalSet = new LongHashSet(); + final Set normalSet = new HashSet<>(); long currentSeq = between(10_000_000, 1_000_000_000); final int iterations = scaledRandomIntBetween(100, 2000); for (long i = 0; i < iterations; i++) { diff --git a/server/src/test/java/org/opensearch/index/translog/SnapshotMatchers.java b/server/src/test/java/org/opensearch/index/translog/SnapshotMatchers.java index 2d15eb3ae8497..dbca53640d9fd 100644 --- a/server/src/test/java/org/opensearch/index/translog/SnapshotMatchers.java +++ b/server/src/test/java/org/opensearch/index/translog/SnapshotMatchers.java @@ -32,8 +32,6 @@ package org.opensearch.index.translog; -import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongSet; import org.opensearch.OpenSearchException; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -42,7 +40,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; public final class SnapshotMatchers { @@ -217,7 +217,7 @@ static class ContainingSeqNoRangeMatcher extends TypeSafeMatcher seqNoList = new HashSet<>(); Translog.Operation op; while ((op = snapshot.next()) != null) { seqNoList.add(op.seqNo()); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java index 9307a70d57ca2..d9a83549f8a97 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java @@ -45,8 +45,6 @@ import org.opensearch.search.aggregations.bucket.range.BinaryRangeAggregator.SortedSetRangeLeafCollector; import org.opensearch.test.OpenSearchTestCase; -import com.carrotsearch.hppc.LongHashSet; - public class BinaryRangeAggregatorTests extends OpenSearchTestCase { private static class FakeSortedSetDocValues extends AbstractSortedSetDocValues { @@ -121,12 +119,12 @@ protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws I final int[] expectedCounts = new int[ranges.length]; final int maxDoc = randomIntBetween(5, 10); for (int doc = 0; doc < maxDoc; ++doc) { - LongHashSet ordinalSet = new LongHashSet(); + Set ordinalSet = new HashSet<>(); final int numValues = randomInt(maxNumValuesPerDoc); while (ordinalSet.size() < numValues) { - ordinalSet.add(random().nextInt(terms.length)); + ordinalSet.add(TestUtil.nextLong(random(), 0, terms.length - 1)); } - final long[] ords = ordinalSet.toArray(); + final long[] ords = ordinalSet.stream().mapToLong(Long::longValue).toArray(); Arrays.sort(ords); values.ords = ords; @@ -222,12 +220,12 @@ protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws I final int[] expectedCounts = new int[ranges.length]; final int maxDoc = randomIntBetween(5, 10); for (int doc = 0; doc < maxDoc; ++doc) { - LongHashSet ordinalSet = new LongHashSet(); + Set ordinalSet = new HashSet<>(); final int numValues = randomInt(maxNumValuesPerDoc); while (ordinalSet.size() < numValues) { - ordinalSet.add(random().nextInt(terms.length)); + ordinalSet.add(TestUtil.nextLong(random(), 0, terms.length - 1)); } - final long[] ords = ordinalSet.toArray(); + final long[] ords = ordinalSet.stream().mapToLong(Long::longValue).toArray(); Arrays.sort(ords); values.ords = ords; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java index 52a32e6e134cc..036bb39e790ae 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -32,11 +32,6 @@ package org.opensearch.search.aggregations.metrics; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.ObjectIntMap; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.ObjectObjectMap; - import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Strings; @@ -54,7 +49,9 @@ import org.opensearch.test.geo.RandomGeoGenerator; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -79,8 +76,8 @@ public abstract class AbstractGeoTestCase extends OpenSearchIntegTestCase { protected static GeoPoint[] singleValues, multiValues; protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, singleCentroid, multiCentroid, unmappedCentroid; - protected static ObjectIntMap expectedDocCountsForGeoHash = null; - protected static ObjectObjectMap expectedCentroidsForGeoHash = null; + protected static Map expectedDocCountsForGeoHash = null; + protected static Map expectedCentroidsForGeoHash = null; protected static final double GEOHASH_TOLERANCE = 1E-5D; @Override @@ -109,8 +106,8 @@ public void setupSuiteScopeCluster() throws Exception { numDocs = randomIntBetween(6, 20); numUniqueGeoPoints = randomIntBetween(1, numDocs); - expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); - expectedCentroidsForGeoHash = new ObjectObjectHashMap<>(numDocs * 2); + expectedDocCountsForGeoHash = new HashMap<>(numDocs * 2); + expectedCentroidsForGeoHash = new HashMap<>(numDocs * 2); singleValues = new GeoPoint[numUniqueGeoPoints]; for (int i = 0; i < singleValues.length; i++) { diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java index ce47862c9884b..5968605fc0c34 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java @@ -32,12 +32,12 @@ package org.opensearch.search.aggregations.metrics; -import com.carrotsearch.hppc.BitMixer; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.breaker.NoopCircuitBreaker; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.BitMixer; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.CoreMatchers; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java index 0726621855b47..6f5f7494331a5 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java @@ -32,15 +32,16 @@ package org.opensearch.search.aggregations.metrics; -import com.carrotsearch.hppc.BitMixer; -import com.carrotsearch.hppc.IntHashSet; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.breaker.NoopCircuitBreaker; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.BitMixer; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import static org.opensearch.search.aggregations.metrics.AbstractHyperLogLog.MAX_PRECISION; @@ -80,7 +81,7 @@ public void testAccuracy() { final int numValues = randomIntBetween(1, 100000); final int maxValue = randomIntBetween(1, randomBoolean() ? 1000 : 100000); final int p = randomIntBetween(14, MAX_PRECISION); - IntHashSet set = new IntHashSet(); + final Set set = new HashSet<>(); HyperLogLogPlusPlus e = new HyperLogLogPlusPlus(p, BigArrays.NON_RECYCLING_INSTANCE, 1); for (int i = 0; i < numValues; ++i) { final int n = randomInt(maxValue); diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java index 8a513d9f45dde..a95533bb3c33c 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.metrics; -import com.carrotsearch.hppc.BitMixer; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.BitMixer; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.lease.Releasables; diff --git a/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java b/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java index 2bf1225aa9b11..86dc27d73d21f 100644 --- a/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java +++ b/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java @@ -32,7 +32,6 @@ package org.opensearch.search.slice; -import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -49,6 +48,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.NumericUtils; import org.opensearch.common.UUIDs; +import org.opensearch.common.util.BitMixer; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java index 8cfc11cb6f32c..88de234d3449b 100644 --- a/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -32,7 +32,6 @@ package org.opensearch.snapshots; -import com.carrotsearch.hppc.IntHashSet; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterState; @@ -72,6 +71,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -457,7 +457,7 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String final Index index = indexMetadata.getIndex(); final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable()); - routingTable.add(IndexRoutingTable.builder(index).initializeAsNewRestore(indexMetadata, recoverySource, new IntHashSet()).build()); + routingTable.add(IndexRoutingTable.builder(index).initializeAsNewRestore(indexMetadata, recoverySource, new HashSet<>()).build()); final RestoreInProgress.Builder restores = new RestoreInProgress.Builder( currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY) diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java index 72434b33ae39f..3d3cff6f3be9b 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java @@ -31,7 +31,6 @@ package org.opensearch.cluster.coordination; -import com.carrotsearch.hppc.LongObjectHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.FixedBitSet; @@ -498,7 +497,7 @@ void unlift() { */ private static class Cache { private final Map> largeMap = new HashMap<>(); - private final LongObjectHashMap> smallMap = new LongObjectHashMap<>(); + private final Map> smallMap = new HashMap<>(); private final Map internalizeStateMap = new HashMap<>(); private final Map, Set> statePermutations = new HashMap<>(); @@ -517,12 +516,11 @@ private boolean addInternal(Object state, FixedBitSet bitSet) { } private boolean addSmall(Object state, long bits) { - int index = smallMap.indexOf(bits); - Set states; - if (index < 0) { - states = Collections.singleton(state); + Set states = smallMap.get(bits); + if (states == null) { + states = Set.of(state); } else { - Set oldStates = smallMap.indexGet(index); + Set oldStates = states; if (oldStates.contains(state)) return false; states = new HashSet<>(oldStates.size() + 1); states.addAll(oldStates); @@ -532,12 +530,7 @@ private boolean addSmall(Object state, long bits) { // Get a unique set object per state permutation. We assume that the number of permutations of states are small. // We thus avoid the overhead of the set data structure. states = statePermutations.computeIfAbsent(states, k -> k); - - if (index < 0) { - smallMap.indexInsert(index, bits, states); - } else { - smallMap.indexReplace(index, states); - } + smallMap.put(bits, states); return true; } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 5134c8dd06fbb..e062c5d166f12 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -31,8 +31,6 @@ package org.opensearch.test; -import com.carrotsearch.hppc.ObjectLongMap; -import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.SeedUtils; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; @@ -60,7 +58,6 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.DiskThresholdSettings; @@ -1492,14 +1489,14 @@ public void assertSeqNos() throws Exception { assertBusy(() -> { final ClusterState state = clusterService().state(); for (final IndexRoutingTable indexRoutingTable : state.routingTable().indicesRouting().values()) { - for (IntObjectCursor indexShardRoutingTable : indexRoutingTable.shards()) { - ShardRouting primaryShardRouting = indexShardRoutingTable.value.primaryShard(); + for (var indexShardRoutingTable : indexRoutingTable.shards().values()) { + ShardRouting primaryShardRouting = indexShardRoutingTable.primaryShard(); final IndexShard primaryShard = getShardOrNull(state, primaryShardRouting); if (primaryShard == null) { continue; // just ignore - shard movement } final SeqNoStats primarySeqNoStats; - final ObjectLongMap syncGlobalCheckpoints; + final Map syncGlobalCheckpoints; try { primarySeqNoStats = primaryShard.seqNoStats(); syncGlobalCheckpoints = primaryShard.getInSyncGlobalCheckpoints(); @@ -1511,7 +1508,7 @@ public void assertSeqNos() throws Exception { primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO)) ); - for (ShardRouting replicaShardRouting : indexShardRoutingTable.value.replicaShards()) { + for (ShardRouting replicaShardRouting : indexShardRoutingTable.replicaShards()) { final IndexShard replicaShard = getShardOrNull(state, replicaShardRouting); if (replicaShard == null) { continue; // just ignore - shard movement @@ -1544,8 +1541,8 @@ public void assertSameDocIdsOnShards() throws Exception { assertBusy(() -> { ClusterState state = client().admin().cluster().prepareState().get().getState(); for (final IndexRoutingTable indexRoutingTable : state.routingTable().indicesRouting().values()) { - for (IntObjectCursor indexShardRoutingTable : indexRoutingTable.shards()) { - ShardRouting primaryShardRouting = indexShardRoutingTable.value.primaryShard(); + for (var indexShardRoutingTable : indexRoutingTable.shards().values()) { + ShardRouting primaryShardRouting = indexShardRoutingTable.primaryShard(); IndexShard primaryShard = getShardOrNull(state, primaryShardRouting); if (primaryShard == null) { continue; @@ -1556,7 +1553,7 @@ public void assertSameDocIdsOnShards() throws Exception { } catch (AlreadyClosedException ex) { continue; } - for (ShardRouting replicaShardRouting : indexShardRoutingTable.value.replicaShards()) { + for (ShardRouting replicaShardRouting : indexShardRoutingTable.replicaShards()) { IndexShard replicaShard = getShardOrNull(state, replicaShardRouting); if (replicaShard == null) { continue; From 88d3ccd2c4ff6c882be5f85437e96be5dcedc31d Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 18 Jul 2023 13:53:43 +0530 Subject: [PATCH 133/186] Do not trigger storeDirectory.sync in segment download flow from remote store (#8739) Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- server/src/main/java/org/opensearch/index/shard/IndexShard.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0eede0cdc4c8d..e6a394b2108e8 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4772,7 +4772,6 @@ private String copySegmentFiles( long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - storeDirectory.sync(Collections.singleton(file)); downloadedSegments.add(file); } else { skippedSegments.add(file); From 2ba1157947c84418234386ad5671719a99f4b889 Mon Sep 17 00:00:00 2001 From: Ashish Date: Tue, 18 Jul 2023 15:15:25 +0530 Subject: [PATCH 134/186] Use CloseableRetryableRefreshListener to drain ongoing after refresh tasks on relocation (#8683) * Use CloesableRetryableRefreshListener to drain ongoing after refresh tasks on relocation --------- Signed-off-by: Ashish Singh --- .../shard/CheckpointRefreshListener.java | 10 +- .../CloseableRetryableRefreshListener.java | 143 ++++++++ .../opensearch/index/shard/IndexShard.java | 11 +- .../shard/RemoteStoreRefreshListener.java | 99 +++--- .../org/opensearch/threadpool/ThreadPool.java | 8 +- ...loseableRetryableRefreshListenerTests.java | 307 ++++++++++++++++++ .../threadpool/ScalingThreadPoolTests.java | 2 +- .../index/shard/IndexShardTestCase.java | 3 +- 8 files changed, 514 insertions(+), 69 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java create mode 100644 server/src/test/java/org/opensearch/index/shard/CloseableRetryableRefreshListenerTests.java diff --git a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java index 4254586f3d70e..357b6c2eaa456 100644 --- a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java @@ -21,7 +21,7 @@ * * @opensearch.internal */ -public class CheckpointRefreshListener implements ReferenceManager.RefreshListener { +public class CheckpointRefreshListener extends CloseableRetryableRefreshListener { protected static Logger logger = LogManager.getLogger(CheckpointRefreshListener.class); @@ -39,12 +39,18 @@ public void beforeRefresh() throws IOException { } @Override - public void afterRefresh(boolean didRefresh) throws IOException { + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { if (didRefresh && shard.state() == IndexShardState.STARTED && shard.getReplicationTracker().isPrimaryMode() && !shard.indexSettings.isSegRepWithRemoteEnabled()) { publisher.publish(shard, shard.getLatestReplicationCheckpoint()); } + return true; + } + + @Override + protected Logger getLogger() { + return logger; } } diff --git a/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java new file mode 100644 index 0000000000000..10e3e04033da3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java @@ -0,0 +1,143 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.ReferenceManager; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.threadpool.ThreadPool; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * RefreshListener that runs afterRefresh method if and only if there is a permit available. Once the listener + * is closed, all the permits are acquired and there are no available permits to afterRefresh. This abstract class provides + * necessary abstract methods to schedule retry. + */ +public abstract class CloseableRetryableRefreshListener implements ReferenceManager.RefreshListener, Closeable { + + /** + * Total permits = 1 ensures that there is only single instance of performAfterRefresh that is running at a time. + * In case there are use cases where concurrency is required, the total permit variable can be put inside the ctor. + */ + private static final int TOTAL_PERMITS = 1; + + private final Semaphore semaphore = new Semaphore(TOTAL_PERMITS); + + private final ThreadPool threadPool; + + /** + * This boolean is used to ensure that there is only 1 retry scheduled/running at any time. + */ + private final AtomicBoolean retryScheduled = new AtomicBoolean(false); + + public CloseableRetryableRefreshListener() { + this.threadPool = null; + } + + public CloseableRetryableRefreshListener(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + @Override + public final void afterRefresh(boolean didRefresh) throws IOException { + boolean successful; + boolean permitAcquired = semaphore.tryAcquire(); + try { + successful = permitAcquired && performAfterRefresh(didRefresh, false); + } finally { + if (permitAcquired) { + semaphore.release(); + } + } + scheduleRetry(successful, didRefresh, permitAcquired); + } + + protected String getRetryThreadPoolName() { + return null; + } + + protected TimeValue getNextRetryInterval() { + return null; + } + + private void scheduleRetry(TimeValue interval, String retryThreadPoolName, boolean didRefresh, boolean isRetry) { + if (this.threadPool == null + || interval == null + || retryThreadPoolName == null + || ThreadPool.THREAD_POOL_TYPES.containsKey(retryThreadPoolName) == false + || interval == TimeValue.MINUS_ONE + || retryScheduled.compareAndSet(false, true) == false) { + return; + } + boolean scheduled = false; + try { + this.threadPool.schedule(() -> { + boolean successful; + boolean permitAcquired = semaphore.tryAcquire(); + try { + successful = permitAcquired && performAfterRefresh(didRefresh, isRetry); + } finally { + if (permitAcquired) { + semaphore.release(); + } + retryScheduled.set(false); + } + scheduleRetry(successful, didRefresh, isRetry || permitAcquired); + }, interval, retryThreadPoolName); + scheduled = true; + getLogger().info("Scheduled retry with didRefresh={} isRetry={}", didRefresh, isRetry); + } finally { + if (scheduled == false) { + retryScheduled.set(false); + } + } + } + + /** + * Schedules the retry based on the {@code afterRefreshSuccessful} value. + * + * @param afterRefreshSuccessful is sent true if the performAfterRefresh(..) is successful. + * @param didRefresh if the refresh did open a new reference then didRefresh will be true + * @param isRetry if this is a failure or permit was not acquired. + */ + private void scheduleRetry(boolean afterRefreshSuccessful, boolean didRefresh, boolean isRetry) { + if (afterRefreshSuccessful == false) { + scheduleRetry(getNextRetryInterval(), getRetryThreadPoolName(), didRefresh, isRetry); + } + } + + /** + * This method needs to be overridden and be provided with what needs to be run on after refresh. + * + * @param didRefresh true if the refresh opened a new reference + * @param isRetry true if this is a retry attempt + * @return true if a retry is needed else false. + */ + protected abstract boolean performAfterRefresh(boolean didRefresh, boolean isRetry); + + @Override + public final void close() throws IOException { + try { + if (semaphore.tryAcquire(TOTAL_PERMITS, 10, TimeUnit.MINUTES)) { + assert semaphore.availablePermits() == 0; + } else { + throw new RuntimeException("timeout while closing gated refresh listener"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + protected abstract Logger getLogger(); +} diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index e6a394b2108e8..76c1f0ae088c9 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -338,6 +338,8 @@ Runnable getGlobalCheckpointSyncer() { private final boolean isTimeSeriesIndex; private final RemoteRefreshSegmentPressureService remoteRefreshSegmentPressureService; + private final List internalRefreshListener = new ArrayList<>(); + public IndexShard( final ShardRouting shardRouting, final IndexSettings indexSettings, @@ -812,6 +814,13 @@ public void relocated( if (syncTranslog) { maybeSync(); } + + // Ensures all in-flight remote store operations drain, before we perform the handoff. + internalRefreshListener.stream() + .filter(refreshListener -> refreshListener instanceof Closeable) + .map(refreshListener -> (Closeable) refreshListener) + .close(); + // no shard operation permits are being held here, move state from started to relocated assert indexShardOperationPermits.getActiveOperationsCount() == OPERATIONS_BLOCKED : "in-flight operations in progress while moving shard state to relocated"; @@ -3659,7 +3668,7 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro } }; - final List internalRefreshListener = new ArrayList<>(); + internalRefreshListener.clear(); internalRefreshListener.add(new RefreshMetricUpdater(refreshMetric)); if (this.checkpointPublisher != null && shardRouting.primary() && indexSettings.isSegRepLocalEnabled()) { internalRefreshListener.add(new CheckpointRefreshListener(this, this.checkpointPublisher)); diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 8f520c0e007bb..2d168839258dd 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -14,7 +14,6 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; @@ -37,7 +36,6 @@ import org.opensearch.index.translog.Translog; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; -import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -51,8 +49,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -63,7 +59,7 @@ * * @opensearch.internal */ -public final class RemoteStoreRefreshListener implements ReferenceManager.RefreshListener { +public final class RemoteStoreRefreshListener extends CloseableRetryableRefreshListener { private final Logger logger; @@ -98,16 +94,8 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres private final RemoteRefreshSegmentTracker segmentTracker; private final Map localSegmentChecksumMap; private long primaryTerm; - - /** - * This boolean is used to ensure that there is only 1 retry scheduled/running at any time. - */ - private final AtomicBoolean retryScheduled = new AtomicBoolean(false); - private volatile Iterator backoffDelayIterator; - private volatile Scheduler.ScheduledCancellable scheduledCancellableRetry; - /** * Keeps track of segment files and their size in bytes which are part of the most recent refresh. */ @@ -122,6 +110,7 @@ public RemoteStoreRefreshListener( SegmentReplicationCheckpointPublisher checkpointPublisher, RemoteRefreshSegmentTracker segmentTracker ) { + super(indexShard.getThreadPool()); logger = Loggers.getLogger(getClass(), indexShard.shardId()); this.indexShard = indexShard; this.storeDirectory = indexShard.store().directory(); @@ -172,32 +161,40 @@ public void beforeRefresh() throws IOException {} * This method also uploads remote_segments_metadata file which contains metadata of each segment file uploaded. * * @param didRefresh true if the refresh opened a new reference + * @return true if the method runs successfully. */ @Override - public void afterRefresh(boolean didRefresh) { + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + if (didRefresh && isRetry == false) { + updateLocalRefreshTimeAndSeqNo(); + } + boolean successful; if (this.primaryTerm != indexShard.getOperationPrimaryTerm() || didRefresh || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty()) { - updateLocalRefreshTimeAndSeqNo(); - try { - indexShard.getThreadPool().executor(ThreadPool.Names.REMOTE_REFRESH).submit(() -> syncSegments(false)).get(); - } catch (InterruptedException | ExecutionException e) { - logger.info("Exception occurred while scheduling syncSegments", e); - } + successful = syncSegments(); + } else { + successful = true; } + return successful; } - private synchronized void syncSegments(boolean isRetry) { - if (indexShard.getReplicationTracker().isPrimaryMode() == false) { - return; + private synchronized boolean syncSegments() { + if (indexShard.getReplicationTracker().isPrimaryMode() == false || indexShard.state() == IndexShardState.CLOSED) { + logger.info( + "Skipped syncing segments with primaryMode={} indexShardState={}", + indexShard.getReplicationTracker().isPrimaryMode(), + indexShard.state() + ); + return true; } ReplicationCheckpoint checkpoint = indexShard.getLatestReplicationCheckpoint(); indexShard.onCheckpointPublished(checkpoint); - beforeSegmentsSync(isRetry); + beforeSegmentsSync(); long refreshTimeMs = segmentTracker.getLocalRefreshTimeMs(), refreshClockTimeMs = segmentTracker.getLocalRefreshClockTimeMs(); long refreshSeqNo = segmentTracker.getLocalRefreshSeqNo(); long bytesBeforeUpload = segmentTracker.getUploadBytesSucceeded(), startTimeInNS = System.nanoTime(); - final AtomicBoolean shouldRetry = new AtomicBoolean(true); + final AtomicBoolean successful = new AtomicBoolean(false); try { if (this.primaryTerm != indexShard.getOperationPrimaryTerm()) { @@ -265,7 +262,7 @@ public void onResponse(Void unused) { ); // At this point since we have uploaded new segments, segment infos and segment metadata file, // along with marking minSeqNoToKeep, upload has succeeded completely. - shouldRetry.set(false); + successful.set(true); } catch (Exception e) { // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried // in the next refresh. This should not affect durability of the indexed data after remote trans-log @@ -295,8 +292,10 @@ public void onFailure(Exception e) { } catch (Throwable t) { logger.error("Exception in RemoteStoreRefreshListener.afterRefresh()", t); } - updateFinalStatusInSegmentTracker(shouldRetry.get() == false, bytesBeforeUpload, startTimeInNS); - afterSegmentsSync(isRetry, shouldRetry.get()); + updateFinalStatusInSegmentTracker(successful.get(), bytesBeforeUpload, startTimeInNS); + // If there are failures in uploading segments, then we should retry as search idle can lead to + // refresh not occurring until write happens. + return successful.get(); } /** @@ -312,10 +311,7 @@ private void clearStaleFilesFromLocalSegmentChecksumMap(Collection local .forEach(localSegmentChecksumMap::remove); } - private void beforeSegmentsSync(boolean isRetry) { - if (isRetry) { - logger.info("Retrying to sync the segments to remote store"); - } + private void beforeSegmentsSync() { // Start tracking total uploads started segmentTracker.incrementTotalUploadsStarted(); } @@ -333,28 +329,12 @@ private void onSuccessfulSegmentsSync( updateRemoteRefreshTimeAndSeqNo(refreshTimeMs, refreshClockTimeMs, refreshSeqNo); // Reset the backoffDelayIterator for the future failures resetBackOffDelayIterator(); - // Cancel the scheduled cancellable retry if possible and set it to null - cancelAndResetScheduledCancellableRetry(); // Set the minimum sequence number for keeping translog indexShard.getEngine().translogManager().setMinSeqNoToKeep(lastRefreshedCheckpoint + 1); // Publishing the new checkpoint which is used for remote store + segrep indexes checkpointPublisher.publish(indexShard, checkpoint); } - /** - * Cancels the scheduled retry if there is one scheduled, and it has not started yet. Clears the reference as the - * schedule retry has been cancelled, or it was null in the first place, or it is running/ran already. - */ - private void cancelAndResetScheduledCancellableRetry() { - if (scheduledCancellableRetry != null && scheduledCancellableRetry.getDelay(TimeUnit.NANOSECONDS) > 0) { - scheduledCancellableRetry.cancel(); - // Since we are cancelling the retry attempt as an internal/external refresh happened already before the retry job could be - // started and the current run successfully uploaded the segments. - retryScheduled.set(false); - } - scheduledCancellableRetry = null; - } - /** * Resets the backoff delay iterator so that the next set of failures starts with the base delay and goes upto max delay. */ @@ -362,18 +342,14 @@ private void resetBackOffDelayIterator() { backoffDelayIterator = EXPONENTIAL_BACKOFF_POLICY.iterator(); } - private void afterSegmentsSync(boolean isRetry, boolean shouldRetry) { - // If this was a retry attempt, then we set the retryScheduled to false so that the next retry (if needed) can be scheduled - if (isRetry) { - retryScheduled.set(false); - } + @Override + protected TimeValue getNextRetryInterval() { + return backoffDelayIterator.next(); + } - // If there are failures in uploading segments, then we should retry as search idle can lead to - // refresh not occurring until write happens. - if (shouldRetry && indexShard.state() != IndexShardState.CLOSED && retryScheduled.compareAndSet(false, true)) { - scheduledCancellableRetry = indexShard.getThreadPool() - .schedule(() -> this.syncSegments(true), backoffDelayIterator.next(), ThreadPool.Names.REMOTE_REFRESH); - } + @Override + protected String getRetryThreadPoolName() { + return ThreadPool.Names.REMOTE_REFRESH_RETRY; } private boolean isRefreshAfterCommit() throws IOException { @@ -522,4 +498,9 @@ private void updateFinalStatusInSegmentTracker(boolean uploadStatus, long bytesB segmentTracker.incrementTotalUploadsFailed(); } } + + @Override + protected Logger getLogger() { + return logger; + } } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 99bb7f95a85e5..0851677bcb13a 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -112,7 +112,7 @@ public static class Names { public static final String TRANSLOG_TRANSFER = "translog_transfer"; public static final String TRANSLOG_SYNC = "translog_sync"; public static final String REMOTE_PURGE = "remote_purge"; - public static final String REMOTE_REFRESH = "remote_refresh"; + public static final String REMOTE_REFRESH_RETRY = "remote_refresh_retry"; public static final String INDEX_SEARCHER = "index_searcher"; } @@ -181,7 +181,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.TRANSLOG_TRANSFER, ThreadPoolType.SCALING); map.put(Names.TRANSLOG_SYNC, ThreadPoolType.FIXED); map.put(Names.REMOTE_PURGE, ThreadPoolType.SCALING); - map.put(Names.REMOTE_REFRESH, ThreadPoolType.SCALING); + map.put(Names.REMOTE_REFRESH_RETRY, ThreadPoolType.SCALING); if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { map.put(Names.INDEX_SEARCHER, ThreadPoolType.RESIZABLE); } @@ -264,8 +264,8 @@ public ThreadPool( builders.put(Names.TRANSLOG_SYNC, new FixedExecutorBuilder(settings, Names.TRANSLOG_SYNC, allocatedProcessors * 4, 10000)); builders.put(Names.REMOTE_PURGE, new ScalingExecutorBuilder(Names.REMOTE_PURGE, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5))); builders.put( - Names.REMOTE_REFRESH, - new ScalingExecutorBuilder(Names.REMOTE_REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)) + Names.REMOTE_REFRESH_RETRY, + new ScalingExecutorBuilder(Names.REMOTE_REFRESH_RETRY, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)) ); if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { builders.put( diff --git a/server/src/test/java/org/opensearch/index/shard/CloseableRetryableRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/CloseableRetryableRefreshListenerTests.java new file mode 100644 index 0000000000000..b9df9ed5a13d8 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/shard/CloseableRetryableRefreshListenerTests.java @@ -0,0 +1,307 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; + +public class CloseableRetryableRefreshListenerTests extends OpenSearchTestCase { + + private static final Logger logger = LogManager.getLogger(CloseableRetryableRefreshListenerTests.class); + + private ThreadPool threadPool; + + @Before + public void init() { + threadPool = new TestThreadPool(getTestName()); + } + + /** + * This tests that the performAfterRefresh method is being invoked when the afterRefresh method is invoked. We check that the countDownLatch is decreasing as intended to validate that the performAfterRefresh is being invoked. + */ + public void testPerformAfterRefresh() throws IOException { + + CountDownLatch countDownLatch = new CountDownLatch(2); + CloseableRetryableRefreshListener testRefreshListener = new CloseableRetryableRefreshListener(null) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return false; + } + + @Override + public void beforeRefresh() {} + + @Override + protected Logger getLogger() { + return logger; + } + }; + + // First invocation of afterRefresh method + testRefreshListener.afterRefresh(true); + assertEquals(1, countDownLatch.getCount()); + + // Second invocation of afterRefresh method + testRefreshListener.afterRefresh(true); + assertEquals(0, countDownLatch.getCount()); + testRefreshListener.close(); + } + + /** + * This tests that close is acquiring all permits and even if the afterRefresh method is called, it is no-op. + */ + public void testCloseAfterRefresh() throws IOException { + final int initialCount = randomIntBetween(10, 100); + final CountDownLatch countDownLatch = new CountDownLatch(initialCount); + CloseableRetryableRefreshListener testRefreshListener = new CloseableRetryableRefreshListener(null) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return false; + } + + @Override + public void beforeRefresh() {} + + @Override + protected Logger getLogger() { + return logger; + } + }; + + int refreshCount = randomIntBetween(1, initialCount); + for (int i = 0; i < refreshCount; i++) { + testRefreshListener.afterRefresh(true); + } + assertEquals(initialCount - refreshCount, countDownLatch.getCount()); + + // Closing the refresh listener so that no further afterRefreshes are executed going forward + testRefreshListener.close(); + + for (int i = 0; i < initialCount - refreshCount; i++) { + testRefreshListener.afterRefresh(true); + } + assertEquals(initialCount - refreshCount, countDownLatch.getCount()); + } + + /** + * This tests that the retry does not get triggered when there are missing configurations or method overrides that empowers the retry to happen. + */ + public void testNoRetry() throws IOException { + int initialCount = randomIntBetween(10, 100); + final CountDownLatch countDownLatch = new CountDownLatch(initialCount); + CloseableRetryableRefreshListener testRefreshListener = new CloseableRetryableRefreshListener(null) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return countDownLatch.getCount() == 0; + } + + @Override + public void beforeRefresh() {} + + @Override + protected Logger getLogger() { + return logger; + } + }; + testRefreshListener.afterRefresh(true); + assertEquals(initialCount - 1, countDownLatch.getCount()); + testRefreshListener.close(); + + testRefreshListener = new CloseableRetryableRefreshListener(threadPool) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return countDownLatch.getCount() == 0; + } + + @Override + public void beforeRefresh() {} + + @Override + protected Logger getLogger() { + return logger; + } + }; + testRefreshListener.afterRefresh(true); + assertEquals(initialCount - 2, countDownLatch.getCount()); + testRefreshListener.close(); + + testRefreshListener = new CloseableRetryableRefreshListener(threadPool) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return countDownLatch.getCount() == 0; + } + + @Override + public void beforeRefresh() {} + + @Override + protected String getRetryThreadPoolName() { + return ThreadPool.Names.REMOTE_REFRESH_RETRY; + } + + @Override + protected Logger getLogger() { + return logger; + } + }; + testRefreshListener.afterRefresh(true); + assertEquals(initialCount - 3, countDownLatch.getCount()); + testRefreshListener.close(); + + testRefreshListener = new CloseableRetryableRefreshListener(threadPool) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return countDownLatch.getCount() == 0; + } + + @Override + public void beforeRefresh() {} + + @Override + protected TimeValue getNextRetryInterval() { + return TimeValue.timeValueMillis(100); + } + + @Override + protected Logger getLogger() { + return logger; + } + }; + testRefreshListener.afterRefresh(true); + assertEquals(initialCount - 4, countDownLatch.getCount()); + testRefreshListener.close(); + } + + /** + * This tests that retry gets scheduled and executed when the configurations and method overrides are done properly. + */ + public void testRetry() throws Exception { + int initialCount = randomIntBetween(10, 20); + final CountDownLatch countDownLatch = new CountDownLatch(initialCount); + CloseableRetryableRefreshListener testRefreshListener = new CloseableRetryableRefreshListener(threadPool) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return countDownLatch.getCount() == 0; + } + + @Override + public void beforeRefresh() {} + + @Override + protected String getRetryThreadPoolName() { + return ThreadPool.Names.REMOTE_REFRESH_RETRY; + } + + @Override + protected TimeValue getNextRetryInterval() { + return TimeValue.timeValueMillis(100); + } + + @Override + protected Logger getLogger() { + return logger; + } + }; + testRefreshListener.afterRefresh(true); + assertBusy(() -> assertEquals(0, countDownLatch.getCount())); + testRefreshListener.close(); + } + + /** + * This tests that once close method is invoked, then even the retries would become no-op. + */ + public void testCloseWithRetryPending() throws IOException { + int initialCount = randomIntBetween(10, 20); + final CountDownLatch countDownLatch = new CountDownLatch(initialCount); + CloseableRetryableRefreshListener testRefreshListener = new CloseableRetryableRefreshListener(threadPool) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + countDownLatch.countDown(); + return countDownLatch.getCount() == 0; + } + + @Override + public void beforeRefresh() {} + + @Override + protected String getRetryThreadPoolName() { + return ThreadPool.Names.REMOTE_REFRESH_RETRY; + } + + @Override + protected TimeValue getNextRetryInterval() { + return TimeValue.timeValueMillis(100); + } + + @Override + protected Logger getLogger() { + return logger; + } + }; + testRefreshListener.afterRefresh(randomBoolean()); + testRefreshListener.close(); + assertNotEquals(0, countDownLatch.getCount()); + } + + public void testCloseWaitsForAcquiringAllPermits() throws Exception { + final CountDownLatch countDownLatch = new CountDownLatch(1); + CloseableRetryableRefreshListener testRefreshListener = new CloseableRetryableRefreshListener(threadPool) { + @Override + protected boolean performAfterRefresh(boolean didRefresh, boolean isRetry) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + countDownLatch.countDown(); + return false; + } + + @Override + public void beforeRefresh() {} + + @Override + protected Logger getLogger() { + return logger; + } + }; + Thread thread = new Thread(() -> { + try { + testRefreshListener.afterRefresh(randomBoolean()); + } catch (IOException e) { + throw new AssertionError(e); + } + }); + thread.start(); + assertBusy(() -> assertEquals(0, countDownLatch.getCount())); + testRefreshListener.close(); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + terminate(threadPool); + } +} diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java index 33ad845ea647e..92bd15d818bca 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java @@ -135,7 +135,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso sizes.put(ThreadPool.Names.TRANSLOG_TRANSFER, ThreadPool::halfAllocatedProcessorsMaxTen); sizes.put(ThreadPool.Names.TRANSLOG_SYNC, n -> 4 * n); sizes.put(ThreadPool.Names.REMOTE_PURGE, ThreadPool::halfAllocatedProcessorsMaxFive); - sizes.put(ThreadPool.Names.REMOTE_REFRESH, ThreadPool::halfAllocatedProcessorsMaxTen); + sizes.put(ThreadPool.Names.REMOTE_REFRESH_RETRY, ThreadPool::halfAllocatedProcessorsMaxTen); return sizes.get(threadPoolName).apply(numberOfProcessors); } diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index b23c28be14865..93b9742ada0da 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -42,7 +42,6 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.Version; import org.opensearch.action.ActionListener; -import org.opensearch.action.LatchedActionListener; import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.PlainActionFuture; @@ -651,7 +650,7 @@ protected RepositoriesService createRepositoriesService() { BlobStore blobStore = Mockito.mock(BlobStore.class); BlobContainer blobContainer = Mockito.mock(BlobContainer.class); doAnswer(invocation -> { - LatchedActionListener> listener = invocation.getArgument(3); + ActionListener> listener = invocation.getArgument(3); listener.onResponse(new ArrayList<>()); return null; }).when(blobContainer) From 230a06daee9c8484e38d6aad867a98c3c10c2485 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 18 Jul 2023 16:47:47 +0530 Subject: [PATCH 135/186] [Remote Segment Store] Use files from segmentInfosSnapshot in upload (#8487) * Use files from segmentInfosSnapshot in upload --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../RemoteStoreBaseIntegTestCase.java | 9 ++ .../remotestore/RemoteStoreForceMergeIT.java | 139 ++++++++++++++++++ .../remotestore/RemoteStoreStatsIT.java | 12 +- .../shard/RemoteStoreRefreshListener.java | 103 +++++-------- 4 files changed, 185 insertions(+), 78 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 10f01749ab4c5..3c463026281eb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -9,7 +9,9 @@ package org.opensearch.remotestore; import org.junit.After; +import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexModule; @@ -49,6 +51,13 @@ public Settings indexSettings() { return defaultIndexSettings(); } + IndexResponse indexSingleDoc(String indexName) { + return client().prepareIndex(indexName) + .setId(UUIDs.randomBase64UUID()) + .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) + .get(); + } + private Settings defaultIndexSettings() { return Settings.builder() .put(super.indexSettings()) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java new file mode 100644 index 0000000000000..1e9458069d459 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -0,0 +1,139 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.junit.Before; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 3) +public class RemoteStoreForceMergeIT extends RemoteStoreBaseIntegTestCase { + + private static final String INDEX_NAME = "remote-store-test-idx-1"; + private static final String TOTAL_OPERATIONS = "total-operations"; + private static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockTransportService.TestPlugin.class); + } + + @Before + public void setup() { + setupRepo(); + } + + @Override + public Settings indexSettings() { + return remoteStoreIndexSettings(0); + } + + private Map indexData(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) { + long totalOperations = 0; + long maxSeqNo = -1; + List indexResponseList = new ArrayList<>(); + for (int i = 0; i < numberOfIterations; i++) { + int numberOfOperations = randomIntBetween(20, 50); + for (int j = 0; j < numberOfOperations; j++) { + IndexResponse response = indexSingleDoc(INDEX_NAME); + maxSeqNo = response.getSeqNo(); + indexResponseList.add(response); + } + totalOperations += numberOfOperations; + if (invokeFlush) { + flush(INDEX_NAME); + } else { + refresh(INDEX_NAME); + } + } + if (deletedDocs == -1) { + deletedDocs = totalOperations; + } + int length = indexResponseList.size(); + for (int j = 0; j < deletedDocs; j++) { + maxSeqNo = client().prepareDelete().setIndex(INDEX_NAME).setId(indexResponseList.get(length - j - 1).getId()).get().getSeqNo(); + } + client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).setFlush(flushAfterMerge).get(); + refresh(INDEX_NAME); + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), totalOperations - deletedDocs); + Map indexingStats = new HashMap<>(); + indexingStats.put(TOTAL_OPERATIONS, totalOperations); + indexingStats.put(MAX_SEQ_NO_TOTAL, maxSeqNo); + return indexingStats; + } + + private void verifyRestoredData(Map indexStats, long deletedDocs) { + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(TOTAL_OPERATIONS) - deletedDocs); + IndexResponse response = indexSingleDoc(INDEX_NAME); + assertEquals(indexStats.get(MAX_SEQ_NO_TOTAL) + 1, response.getSeqNo()); + refresh(INDEX_NAME); + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(TOTAL_OPERATIONS) + 1 - deletedDocs); + } + + private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) + throws IOException { + createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + Map indexStats = indexData(numberOfIterations, invokeFlush, flushAfterMerge, deletedDocs); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + ensureGreen(INDEX_NAME); + + if (deletedDocs == -1) { + verifyRestoredData(indexStats, indexStats.get(TOTAL_OPERATIONS)); + } else { + verifyRestoredData(indexStats, deletedDocs); + } + } + + // Following integ tests use randomBoolean to control the number of integ tests. If we use the separate + // values for each of the flags, number of integ tests become 16 in comparison to current 2. + // We have run all the 16 tests on local and they run fine. + public void testRestoreForceMergeSingleIteration() throws IOException { + boolean invokeFLush = randomBoolean(); + boolean flushAfterMerge = randomBoolean(); + testRestoreWithMergeFlow(1, invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); + } + + public void testRestoreForceMergeMultipleIterations() throws IOException { + boolean invokeFLush = randomBoolean(); + boolean flushAfterMerge = randomBoolean(); + testRestoreWithMergeFlow(randomIntBetween(2, 5), invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); + } + + public void testRestoreForceMergeMultipleIterationsDeleteAll() throws IOException { + boolean invokeFLush = randomBoolean(); + boolean flushAfterMerge = randomBoolean(); + testRestoreWithMergeFlow(randomIntBetween(2, 3), invokeFLush, flushAfterMerge, -1); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 0e4774c1f3454..28b01e0ab1ef6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -12,10 +12,8 @@ import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsRequestBuilder; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; -import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.UUIDs; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.test.OpenSearchIntegTestCase; @@ -128,7 +126,7 @@ private void indexDocs() { } int numberOfOperations = randomIntBetween(20, 50); for (int j = 0; j < numberOfOperations; j++) { - indexSingleDoc(); + indexSingleDoc(INDEX_NAME); } } } @@ -149,12 +147,4 @@ private void assertResponseStats(RemoteRefreshSegmentTracker.Stats stats) { assertTrue(stats.uploadBytesPerSecMovingAverage > 0); assertTrue(stats.uploadTimeMovingAverage > 0); } - - private IndexResponse indexSingleDoc() { - return client().prepareIndex(INDEX_NAME) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - } diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 2d168839258dd..3ea8278038ac5 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; @@ -40,13 +39,10 @@ import java.io.IOException; import java.util.Collection; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -216,77 +212,50 @@ private synchronized boolean syncSegments() { long lastRefreshedCheckpoint = ((InternalEngine) indexShard.getEngine()).lastRefreshedCheckpoint(); Collection localSegmentsPostRefresh = segmentInfos.files(true); - List segmentInfosFiles = localSegmentsPostRefresh.stream() - .filter(file -> file.startsWith(IndexFileNames.SEGMENTS)) - .collect(Collectors.toList()); - Optional latestSegmentInfos = segmentInfosFiles.stream() - .max(Comparator.comparingLong(SegmentInfos::generationFromSegmentsFileName)); - - if (latestSegmentInfos.isPresent()) { - // SegmentInfosSnapshot is a snapshot of reader's view of segments and may not contain - // all the segments from last commit if they are merged away but not yet committed. - // Each metadata file in the remote segment store represents a commit and the following - // statement keeps sure that each metadata will always contain all the segments from last commit + refreshed - // segments. - SegmentInfos segmentCommitInfos; - try { - segmentCommitInfos = SegmentInfos.readCommit(storeDirectory, latestSegmentInfos.get()); - } catch (Exception e) { - // Seeing discrepancy in segment infos and files on disk. SegmentInfosSnapshot is returning - // a segment_N file which does not exist on local disk. - logger.error("Exception occurred while SegmentInfos.readCommit(..)", e); - logger.error("segmentInfosFiles={} diskFiles={}", localSegmentsPostRefresh, storeDirectory.listAll()); - throw e; - } - localSegmentsPostRefresh.addAll(segmentCommitInfos.files(true)); - segmentInfosFiles.stream() - .filter(file -> !file.equals(latestSegmentInfos.get())) - .forEach(localSegmentsPostRefresh::remove); - - // Create a map of file name to size and update the refresh segment tracker - updateLocalSizeMapAndTracker(localSegmentsPostRefresh); - CountDownLatch latch = new CountDownLatch(1); - ActionListener segmentUploadsCompletedListener = new LatchedActionListener<>(new ActionListener<>() { - @Override - public void onResponse(Void unused) { - try { - // Start metadata file upload - uploadMetadata(localSegmentsPostRefresh, segmentInfos); - clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); - onSuccessfulSegmentsSync( - refreshTimeMs, - refreshClockTimeMs, - refreshSeqNo, - lastRefreshedCheckpoint, - checkpoint - ); - // At this point since we have uploaded new segments, segment infos and segment metadata file, - // along with marking minSeqNoToKeep, upload has succeeded completely. - successful.set(true); - } catch (Exception e) { - // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried - // in the next refresh. This should not affect durability of the indexed data after remote trans-log - // integration. - logger.warn("Exception in post new segment upload actions", e); - } + // Create a map of file name to size and update the refresh segment tracker + updateLocalSizeMapAndTracker(localSegmentsPostRefresh); + CountDownLatch latch = new CountDownLatch(1); + ActionListener segmentUploadsCompletedListener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + try { + // Start metadata file upload + uploadMetadata(localSegmentsPostRefresh, segmentInfos); + clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); + onSuccessfulSegmentsSync( + refreshTimeMs, + refreshClockTimeMs, + refreshSeqNo, + lastRefreshedCheckpoint, + checkpoint + ); + // At this point since we have uploaded new segments, segment infos and segment metadata file, + // along with marking minSeqNoToKeep, upload has succeeded completely. + successful.set(true); + } catch (Exception e) { + // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried + // as part of exponential back-off retry logic. This should not affect durability of the indexed data + // with remote trans-log integration. + logger.warn("Exception in post new segment upload actions", e); } + } - @Override - public void onFailure(Exception e) { - logger.warn("Exception while uploading new segments to the remote segment store", e); - } - }, latch); + @Override + public void onFailure(Exception e) { + logger.warn("Exception while uploading new segments to the remote segment store", e); + } + }, latch); - // Start the segments files upload - uploadNewSegments(localSegmentsPostRefresh, segmentUploadsCompletedListener); - latch.await(); - } + // Start the segments files upload + uploadNewSegments(localSegmentsPostRefresh, segmentUploadsCompletedListener); + latch.await(); } catch (EngineException e) { logger.warn("Exception while reading SegmentInfosSnapshot", e); } } catch (IOException e) { // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried - // in the next refresh. This should not affect durability of the indexed data after remote trans-log integration. + // as part of exponential back-off retry logic. This should not affect durability of the indexed data + // with remote trans-log integration. logger.warn("Exception while uploading new segments to the remote segment store", e); } } catch (Throwable t) { From 2f830be1589d2162db9e7ad70381eabfd9377c80 Mon Sep 17 00:00:00 2001 From: Poojita Raj Date: Tue, 18 Jul 2023 10:30:12 -0700 Subject: [PATCH 136/186] [Segment Replication][Remote Store] Remove commits when remote store is enabled (#8050) * remove commits + fix failing test Signed-off-by: Poojita Raj * fix failing tests Signed-off-by: Poojita Raj * fix precommit failure Signed-off-by: Poojita Raj * remove logs Signed-off-by: Poojita Raj * address review comments Signed-off-by: Poojita Raj --------- Signed-off-by: Poojita Raj --- .../index/engine/NRTReplicationEngine.java | 30 ++++++++---- .../opensearch/index/shard/IndexShard.java | 47 +++++++++--------- .../engine/NRTReplicationEngineTests.java | 48 +++++++++++++++---- 3 files changed, 83 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index b5552ed552f09..a9f7a2e70884c 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -56,6 +56,7 @@ public class NRTReplicationEngine extends Engine { private final CompletionStatsCache completionStatsCache; private final LocalCheckpointTracker localCheckpointTracker; private final WriteOnlyTranslogManager translogManager; + private final boolean shouldCommit; private volatile long lastReceivedGen = SequenceNumbers.NO_OPS_PERFORMED; @@ -113,6 +114,7 @@ public void onAfterTranslogSync() { engineConfig.getPrimaryModeSupplier() ); this.translogManager = translogManagerRef; + this.shouldCommit = engineConfig.getIndexSettings().isRemoteStoreEnabled() == false; } catch (IOException e) { IOUtils.closeWhileHandlingException(store::decRef, readerManager, translogManagerRef); throw new EngineCreationFailureException(shardId, "failed to create engine", e); @@ -163,7 +165,9 @@ public synchronized void updateSegments(final SegmentInfos infos) throws IOExcep * @throws IOException - When there is an IO error committing the SegmentInfos. */ private void commitSegmentInfos(SegmentInfos infos) throws IOException { - store.commitSegmentInfos(infos, localCheckpointTracker.getMaxSeqNo(), localCheckpointTracker.getProcessedCheckpoint()); + if (shouldCommit) { + store.commitSegmentInfos(infos, localCheckpointTracker.getMaxSeqNo(), localCheckpointTracker.getProcessedCheckpoint()); + } this.lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo(); translogManager.syncTranslog(); } @@ -383,15 +387,21 @@ protected final void closeNoLock(String reason, CountDownLatch closedLatch) { assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() : "Either the write lock must be held or the engine must be currently be failing itself"; try { - final SegmentInfos latestSegmentInfos = getLatestSegmentInfos(); - /* - This is a workaround solution which decreases the chances of conflict on replica nodes when same file is copied - from two different primaries during failover. Increasing counter helps in avoiding this conflict as counter is - used to generate new segment file names. The ideal solution is to identify the counter from previous primary. - */ - latestSegmentInfos.counter = latestSegmentInfos.counter + SI_COUNTER_INCREMENT; - latestSegmentInfos.changed(); - commitSegmentInfos(latestSegmentInfos); + // if remote store is enabled, all segments durably persisted + if (shouldCommit) { + final SegmentInfos latestSegmentInfos = getLatestSegmentInfos(); + /* + This is a workaround solution which decreases the chances of conflict on replica nodes when same file is copied + from two different primaries during failover. Increasing counter helps in avoiding this conflict as counter is + used to generate new segment file names. The ideal solution is to identify the counter from previous primary. + */ + latestSegmentInfos.counter = latestSegmentInfos.counter + SI_COUNTER_INCREMENT; + latestSegmentInfos.changed(); + commitSegmentInfos(latestSegmentInfos); + } else { + store.directory().sync(List.of(store.directory().listAll())); + store.directory().syncMetaData(); + } IOUtils.close(readerManager, translogManager, store::decRef); } catch (Exception e) { logger.warn("failed to close engine", e); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 76c1f0ae088c9..fbb2df7f9640a 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4661,31 +4661,34 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re indexInput, remoteSegmentMetadata.getGeneration() ); + // Replicas never need a local commit if (shouldCommit) { - long processedLocalCheckpoint = Long.parseLong(infosSnapshot.getUserData().get(LOCAL_CHECKPOINT_KEY)); - // Following code block makes sure to use SegmentInfosSnapshot in the remote store if generation differs - // with local filesystem. If local filesystem already has segments_N+2 and infosSnapshot has generation N, - // after commit, there would be 2 files that would be created segments_N+1 and segments_N+2. With the - // policy of preserving only the latest commit, we will delete segments_N+1 which in fact is the part of the latest - // commit. - Optional localMaxSegmentInfos = localSegmentFiles.stream() - .filter(file -> file.startsWith(IndexFileNames.SEGMENTS)) - .max(Comparator.comparingLong(SegmentInfos::generationFromSegmentsFileName)); - if (localMaxSegmentInfos.isPresent() - && infosSnapshot.getGeneration() < SegmentInfos.generationFromSegmentsFileName(localMaxSegmentInfos.get()) - - 1) { - // If remote translog is not enabled, local translog will be created with different UUID. - // This fails in Store.trimUnsafeCommits() as translog UUID of checkpoint and SegmentInfos needs - // to be same. Following code block make sure to have the same UUID. - if (indexSettings.isRemoteTranslogStoreEnabled() == false) { - SegmentInfos localSegmentInfos = store.readLastCommittedSegmentsInfo(); - Map userData = new HashMap<>(infosSnapshot.getUserData()); - userData.put(TRANSLOG_UUID_KEY, localSegmentInfos.userData.get(TRANSLOG_UUID_KEY)); - infosSnapshot.setUserData(userData, false); + if (this.shardRouting.primary()) { + long processedLocalCheckpoint = Long.parseLong(infosSnapshot.getUserData().get(LOCAL_CHECKPOINT_KEY)); + // Following code block makes sure to use SegmentInfosSnapshot in the remote store if generation differs + // with local filesystem. If local filesystem already has segments_N+2 and infosSnapshot has generation N, + // after commit, there would be 2 files that would be created segments_N+1 and segments_N+2. With the + // policy of preserving only the latest commit, we will delete segments_N+1 which in fact is the part of the + // latest commit. + Optional localMaxSegmentInfos = localSegmentFiles.stream() + .filter(file -> file.startsWith(IndexFileNames.SEGMENTS)) + .max(Comparator.comparingLong(SegmentInfos::generationFromSegmentsFileName)); + if (localMaxSegmentInfos.isPresent() + && infosSnapshot.getGeneration() < SegmentInfos.generationFromSegmentsFileName(localMaxSegmentInfos.get()) + - 1) { + // If remote translog is not enabled, local translog will be created with different UUID. + // This fails in Store.trimUnsafeCommits() as translog UUID of checkpoint and SegmentInfos needs + // to be same. Following code block make sure to have the same UUID. + if (indexSettings.isRemoteTranslogStoreEnabled() == false) { + SegmentInfos localSegmentInfos = store.readLastCommittedSegmentsInfo(); + Map userData = new HashMap<>(infosSnapshot.getUserData()); + userData.put(TRANSLOG_UUID_KEY, localSegmentInfos.userData.get(TRANSLOG_UUID_KEY)); + infosSnapshot.setUserData(userData, false); + } + storeDirectory.deleteFile(localMaxSegmentInfos.get()); } - storeDirectory.deleteFile(localMaxSegmentInfos.get()); + store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint); } - store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint); } else { finalizeReplication(infosSnapshot); } diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java index e25e6ea206d84..64fe42493c686 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java @@ -47,6 +47,14 @@ public class NRTReplicationEngineTests extends EngineTestCase { Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build() ); + private static final IndexSettings REMOTE_STORE_INDEX_SETTINGS = IndexSettingsModule.newIndexSettings( + "index", + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build() + ); + public void testCreateEngine() throws IOException { final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); try ( @@ -132,6 +140,29 @@ public void testUpdateSegments_replicaReceivesSISWithHigherGen() throws IOExcept } } + public void testUpdateSegments_replicaReceivesSISWithHigherGen_remoteStoreEnabled() throws IOException { + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + + try ( + final Store nrtEngineStore = createStore(REMOTE_STORE_INDEX_SETTINGS, newDirectory()); + final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore, REMOTE_STORE_INDEX_SETTINGS) + ) { + // assume we start at the same gen. + assertEquals(2, nrtEngine.getLatestSegmentInfos().getGeneration()); + assertEquals(nrtEngine.getLatestSegmentInfos().getGeneration(), nrtEngine.getLastCommittedSegmentInfos().getGeneration()); + assertEquals(engine.getLatestSegmentInfos().getGeneration(), nrtEngine.getLatestSegmentInfos().getGeneration()); + + // flush the primary engine - we don't need any segments, just force a new commit point. + engine.flush(true, true); + assertEquals(3, engine.getLatestSegmentInfos().getGeneration()); + + // When remote store is enabled, we don't commit on replicas since all segments are durably persisted in the store + nrtEngine.updateSegments(engine.getLatestSegmentInfos()); + assertEquals(2, nrtEngine.getLastCommittedSegmentInfos().getGeneration()); + assertEquals(2, nrtEngine.getLatestSegmentInfos().getGeneration()); + } + } + public void testUpdateSegments_replicaReceivesSISWithLowerGen() throws IOException { // if the replica is already at segments_N that is received, it will commit segments_N+1. final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); @@ -315,18 +346,11 @@ public void testCommitSegmentInfos() throws Exception { } } - private NRTReplicationEngine buildNrtReplicaEngine(AtomicLong globalCheckpoint, Store store) throws IOException { + private NRTReplicationEngine buildNrtReplicaEngine(AtomicLong globalCheckpoint, Store store, IndexSettings settings) + throws IOException { Lucene.cleanLuceneIndex(store.directory()); final Path translogDir = createTempDir(); - final EngineConfig replicaConfig = config( - defaultSettings, - store, - translogDir, - NoMergePolicy.INSTANCE, - null, - null, - globalCheckpoint::get - ); + final EngineConfig replicaConfig = config(settings, store, translogDir, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get); if (Lucene.indexExists(store.directory()) == false) { store.createEmpty(replicaConfig.getIndexSettings().getIndexVersionCreated().luceneVersion); final String translogUuid = Translog.createEmptyTranslog( @@ -339,4 +363,8 @@ private NRTReplicationEngine buildNrtReplicaEngine(AtomicLong globalCheckpoint, } return new NRTReplicationEngine(replicaConfig); } + + private NRTReplicationEngine buildNrtReplicaEngine(AtomicLong globalCheckpoint, Store store) throws IOException { + return buildNrtReplicaEngine(globalCheckpoint, store, defaultSettings); + } } From 57d5e90cd10033eb67e13731728af25ce76f484d Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 18 Jul 2023 14:09:39 -0400 Subject: [PATCH 137/186] Remove SEARCH_PIPELINE feature flag (#8513) * Remove SEARCH_PIPELINE feature flag Signed-off-by: Louis Chu * Fix gradle check Signed-off-by: Louis Chu * Bugfix on search service implicit execution Signed-off-by: Louis Chu * Set the flag for bwc test Signed-off-by: Louis Chu * Remove feature flag setting on qa Signed-off-by: Louis Chu --------- Signed-off-by: Louis Chu --- client/rest-high-level/build.gradle | 2 -- distribution/src/config/opensearch.yml | 6 ------ modules/search-pipeline-common/build.gradle | 3 --- .../common/SearchPipelineCommonIT.java | 7 ------- qa/mixed-cluster/build.gradle | 1 - qa/smoke-test-multinode/build.gradle | 1 - rest-api-spec/build.gradle | 1 - .../org/opensearch/action/ActionModule.java | 8 +++----- .../common/settings/FeatureFlagSettings.java | 1 - .../opensearch/common/util/FeatureFlags.java | 8 -------- .../org/opensearch/index/IndexSettings.java | 14 +------------ .../main/java/org/opensearch/node/Node.java | 4 +--- .../search/builder/SearchSourceBuilder.java | 20 +++++++++---------- .../pipeline/SearchPipelineService.java | 15 ++------------ .../opensearch/index/IndexSettingsTests.java | 18 ----------------- .../pipeline/SearchPipelineServiceTests.java | 12 ++++------- .../snapshots/SnapshotResiliencyTests.java | 3 +-- .../search/RandomSearchRequestGenerator.java | 3 +-- 18 files changed, 22 insertions(+), 105 deletions(-) diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 7daf4761b9310..770cb3f78ca47 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -106,8 +106,6 @@ testClusters.all { extraConfigFile nodeTrustStore.name, nodeTrustStore extraConfigFile pkiTrustCert.name, pkiTrustCert - // Enable APIs behind feature flags - setting 'opensearch.experimental.feature.search_pipeline.enabled', 'true' } thirdPartyAudit.ignoreMissingClasses( diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 8c4160db98857..99e5fb6f9a226 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -128,12 +128,6 @@ ${path.logs} #opensearch.experimental.feature.extensions.enabled: false # # -# Gates the search pipeline feature. This feature enables configurable processors -# for search requests and search responses, similar to ingest pipelines. -# -#opensearch.experimental.feature.search_pipeline.enabled: false -# -# # Gates the concurrent segment search feature. This feature enables concurrent segment search in a separate # index searcher threadpool. # diff --git a/modules/search-pipeline-common/build.gradle b/modules/search-pipeline-common/build.gradle index fe3c097ff6886..657392d884e97 100644 --- a/modules/search-pipeline-common/build.gradle +++ b/modules/search-pipeline-common/build.gradle @@ -28,9 +28,6 @@ restResources { } } -testClusters.all { - setting 'opensearch.experimental.feature.search_pipeline.enabled', 'true' -} thirdPartyAudit.ignoreMissingClasses( // from log4j diff --git a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java index 1e24476eeff27..faa0859e8e33f 100644 --- a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java +++ b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java @@ -19,8 +19,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.core.common.bytes.BytesArray; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.plugins.Plugin; @@ -35,11 +33,6 @@ @OpenSearchIntegTestCase.SuiteScopeTestCase public class SearchPipelineCommonIT extends OpenSearchIntegTestCase { - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(FeatureFlags.SEARCH_PIPELINE, "true").build(); - } - @Override protected Collection> nodePlugins() { return List.of(SearchPipelineCommonModulePlugin.class); diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index d64bf245dbf8f..822977c55368a 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -65,7 +65,6 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { numberOfNodes = 4 setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - setting 'opensearch.experimental.feature.search_pipeline.enabled', 'true' } } diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index 8f13e4bf8736e..25261f5e3ff7d 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -43,7 +43,6 @@ File repo = file("$buildDir/testclusters/repo") testClusters.integTest { numberOfNodes = 2 setting 'path.repo', repo.absolutePath - setting 'opensearch.experimental.feature.search_pipeline.enabled', 'true' } integTest { diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index dcc62d11568ae..535749a5e0c14 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -28,7 +28,6 @@ artifacts { testClusters.all { module ':modules:mapper-extras' - setting 'opensearch.experimental.feature.search_pipeline.enabled', 'true' } test.enabled = false diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 56d1758161ced..2ce1d4bcd4b02 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -946,11 +946,9 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestDeleteDecommissionStateAction()); // Search pipelines API - if (FeatureFlags.isEnabled(FeatureFlags.SEARCH_PIPELINE)) { - registerHandler.accept(new RestPutSearchPipelineAction()); - registerHandler.accept(new RestGetSearchPipelineAction()); - registerHandler.accept(new RestDeleteSearchPipelineAction()); - } + registerHandler.accept(new RestPutSearchPipelineAction()); + registerHandler.accept(new RestGetSearchPipelineAction()); + registerHandler.accept(new RestDeleteSearchPipelineAction()); // Extensions API if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index dae66c79c63ec..ed01347b115ea 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -38,7 +38,6 @@ protected FeatureFlagSettings( FeatureFlags.REMOTE_STORE_SETTING, FeatureFlags.EXTENSIONS_SETTING, FeatureFlags.IDENTITY_SETTING, - FeatureFlags.SEARCH_PIPELINE_SETTING, FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING, FeatureFlags.TELEMETRY_SETTING ) diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 915f231a1ea49..e2663b56c5cca 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -46,12 +46,6 @@ public class FeatureFlags { */ public static final String EXTENSIONS = "opensearch.experimental.feature.extensions.enabled"; - /** - * Gates the search pipeline features during initial development. - * Once the feature is complete and ready for release, this feature flag can be removed. - */ - public static final String SEARCH_PIPELINE = "opensearch.experimental.feature.search_pipeline.enabled"; - /** * Gates the functionality of identity. */ @@ -106,8 +100,6 @@ public static boolean isEnabled(String featureFlagName) { public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); - public static final Setting SEARCH_PIPELINE_SETTING = Setting.boolSetting(SEARCH_PIPELINE, true, Property.NodeScope); - public static final Setting IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope); public static final Setting TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 5b96b09a65cea..7a369c8edcb2e 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -41,7 +41,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; @@ -64,7 +63,6 @@ import java.util.function.UnaryOperator; import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; -import static org.opensearch.common.util.FeatureFlags.SEARCH_PIPELINE; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; @@ -1625,16 +1623,6 @@ public String getDefaultSearchPipeline() { } public void setDefaultSearchPipeline(String defaultSearchPipeline) { - if (FeatureFlags.isEnabled(SEARCH_PIPELINE)) { - this.defaultSearchPipeline = defaultSearchPipeline; - } else { - throw new SettingsException( - "Unable to update setting: " - + DEFAULT_SEARCH_PIPELINE.getKey() - + ". This is an experimental feature that is currently disabled, please enable the " - + SEARCH_PIPELINE - + " feature flag first." - ); - } + this.defaultSearchPipeline = defaultSearchPipeline; } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index ee89964dde810..24cf822ac6c92 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -258,7 +258,6 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static org.opensearch.common.util.FeatureFlags.SEARCH_PIPELINE; import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; @@ -980,8 +979,7 @@ protected Node( xContentRegistry, namedWriteableRegistry, pluginsService.filterPlugins(SearchPipelinePlugin.class), - client, - FeatureFlags.isEnabled(SEARCH_PIPELINE) + client ); final TaskCancellationMonitoringSettings taskCancellationMonitoringSettings = new TaskCancellationMonitoringSettings( settings, diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index 3cb3e7a142e1d..78cb895a0a4c0 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -36,7 +36,6 @@ import org.opensearch.Version; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -1250,16 +1249,15 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th collapse = CollapseBuilder.fromXContent(parser); } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); - } else if (FeatureFlags.isEnabled(FeatureFlags.SEARCH_PIPELINE) - && SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) { - searchPipelineSource = parser.mapOrdered(); - } else { - throw new ParsingException( - parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + currentFieldName + "].", - parser.getTokenLocation() - ); - } + } else if (SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) { + searchPipelineSource = parser.mapOrdered(); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation() + ); + } } else if (token == XContentParser.Token.START_ARRAY) { if (STORED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { storedFieldsContext = StoredFieldsContext.fromXContent(STORED_FIELDS_FIELD.getPreferredName(), parser); diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index ae02068fbcc12..7384b494d71fc 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -85,8 +85,6 @@ public class SearchPipelineService implements ClusterStateApplier, ReportingServ private final OperationMetrics totalRequestProcessingMetrics = new OperationMetrics(); private final OperationMetrics totalResponseProcessingMetrics = new OperationMetrics(); - private final boolean isEnabled; - public SearchPipelineService( ClusterService clusterService, ThreadPool threadPool, @@ -96,8 +94,7 @@ public SearchPipelineService( NamedXContentRegistry namedXContentRegistry, NamedWriteableRegistry namedWriteableRegistry, List searchPipelinePlugins, - Client client, - boolean isEnabled + Client client ) { this.clusterService = clusterService; this.scriptService = scriptService; @@ -123,7 +120,6 @@ public SearchPipelineService( ); putPipelineTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.PUT_SEARCH_PIPELINE_KEY, true); deletePipelineTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.DELETE_SEARCH_PIPELINE_KEY, true); - this.isEnabled = isEnabled; } private static Map> processorFactories( @@ -233,10 +229,6 @@ public void putPipeline( PutSearchPipelineRequest request, ActionListener listener ) throws Exception { - if (isEnabled == false) { - throw new IllegalArgumentException("Experimental search pipeline feature is not enabled"); - } - validatePipeline(searchPipelineInfos, request); clusterService.submitStateUpdateTask( "put-search-pipeline-" + request.getId(), @@ -371,9 +363,6 @@ static ClusterState innerDelete(DeleteSearchPipelineRequest request, ClusterStat public PipelinedRequest resolvePipeline(SearchRequest searchRequest) { Pipeline pipeline = Pipeline.NO_OP_PIPELINE; - if (isEnabled == false) { - return new PipelinedRequest(pipeline, searchRequest); - } if (searchRequest.source() != null && searchRequest.source().searchPipelineSource() != null) { // Pipeline defined in search request (ad hoc pipeline). if (searchRequest.pipeline() != null) { @@ -401,7 +390,7 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) { if (searchRequest.pipeline() != null) { // Named pipeline specified for the request pipelineId = searchRequest.pipeline(); - } else if (searchRequest.indices() != null && searchRequest.indices().length == 1) { + } else if (state != null && searchRequest.indices() != null && searchRequest.indices().length == 1) { // Check for index default pipeline IndexMetadata indexMetadata = state.metadata().index(searchRequest.indices()[0]); if (indexMetadata != null) { diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 490d54d778477..26e53ebc00b26 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -1110,7 +1110,6 @@ public void testExtendedCompatibilityVersionWithoutFeatureFlag() { @SuppressForbidden(reason = "sets the SEARCH_PIPELINE feature flag") public void testDefaultSearchPipeline() throws Exception { - FeatureFlagSetter.set(FeatureFlags.SEARCH_PIPELINE); IndexMetadata metadata = newIndexMeta( "index", Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() @@ -1127,21 +1126,4 @@ public void testDefaultSearchPipeline() throws Exception { settings.updateIndexMetadata(metadata); assertEquals("foo", settings.getDefaultSearchPipeline()); } - - public void testDefaultSearchPipelineWithoutFeatureFlag() { - IndexMetadata metadata = newIndexMeta( - "index", - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() - ); - IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertEquals(SearchPipelineService.NOOP_PIPELINE_ID, settings.getDefaultSearchPipeline()); - IndexMetadata updatedMetadata = newIndexMeta( - "index", - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexSettings.DEFAULT_SEARCH_PIPELINE.getKey(), "foo") - .build() - ); - assertThrows(SettingsException.class, () -> settings.updateIndexMetadata(updatedMetadata)); - } } diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index 118d77a7a5c41..d0f4a974fc305 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -112,8 +112,7 @@ public void testSearchPipelinePlugin() { this.xContentRegistry(), this.writableRegistry(), List.of(DUMMY_PLUGIN), - client, - false + client ); Map> requestProcessorFactories = searchPipelineService .getRequestProcessorFactories(); @@ -138,8 +137,7 @@ public void testSearchPipelinePluginDuplicate() { this.xContentRegistry(), this.writableRegistry(), List.of(DUMMY_PLUGIN, DUMMY_PLUGIN), - client, - false + client ) ); assertTrue(e.getMessage(), e.getMessage().contains(" already registered")); @@ -156,8 +154,7 @@ public void testResolveSearchPipelineDoesNotExist() { this.xContentRegistry(), this.writableRegistry(), List.of(DUMMY_PLUGIN), - client, - true + client ); final SearchRequest searchRequest = new SearchRequest("_index").pipeline("bar"); IllegalArgumentException e = expectThrows( @@ -386,8 +383,7 @@ public Map> getSearchPhas } }), - client, - true + client ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index e461e457bead1..a121a190096b4 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2063,8 +2063,7 @@ public void onFailure(final Exception e) { namedXContentRegistry, namedWriteableRegistry, List.of(), - client, - false + client ) ) ); diff --git a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java index ae993da34bdf5..e7b1dd2bde1ea 100644 --- a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java @@ -38,7 +38,6 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.text.Text; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -406,7 +405,7 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } builder.pointInTimeBuilder(pit); } - if (FeatureFlags.isEnabled(FeatureFlags.SEARCH_PIPELINE) && randomBoolean()) { + if (randomBoolean()) { builder.searchPipelineSource(new HashMap<>()); } return builder; From 1d3b0067465abf2fbcf4451b6826348e20d1f2c0 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 19 Jul 2023 12:41:47 +0530 Subject: [PATCH 138/186] [Remote Store] Change remote store integ tests to use remote store for replication (#8768) * Change remote store integ tests to use remote store for replication --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- ...emoteStoreMockRepositoryIntegTestCase.java | 1 + .../RemoteStoreBaseIntegTestCase.java | 6 +- .../SegmentReplicationRemoteStoreIT.java | 62 ------------------- 3 files changed, 6 insertions(+), 63 deletions(-) delete mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index f57c312aa2cd0..bb8839a47eb69 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -44,6 +44,7 @@ protected Settings featureFlagSettings() { @Before public void setup() { FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); internalCluster().startClusterManagerOnlyNode(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 3c463026281eb..eb8913591b38b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -44,7 +44,11 @@ protected boolean addMockInternalEngine() { @Override protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); + return Settings.builder() + .put(super.featureFlagSettings()) + .put(FeatureFlags.REMOTE_STORE, "true") + .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") + .build(); } public Settings indexSettings() { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java deleted file mode 100644 index 7e79812fcfeea..0000000000000 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationRemoteStoreIT.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.remotestore; - -import org.junit.After; -import org.junit.Before; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; -import org.opensearch.indices.replication.SegmentReplicationIT; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.nio.file.Path; - -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; - -/** - * The aim of this class is to run Segment Replication integ tests by enabling remote store specific settings. - * This makes sure that the constructs/flows that are being tested with Segment Replication, holds true after enabling - * remote store. - */ -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class SegmentReplicationRemoteStoreIT extends SegmentReplicationIT { - - private static final String REPOSITORY_NAME = "test-remote-store-repo"; - - @Override - public Settings indexSettings() { - return Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) - .build(); - } - - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); - } - - @Before - private void setup() { - internalCluster().startClusterManagerOnlyNode(); - Path absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); - } - - @After - public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); - } -} From 83a4b6e9effc0e73ecba67e5dd9246e06b2c3df5 Mon Sep 17 00:00:00 2001 From: suraj kumar Date: Wed, 19 Jul 2023 21:42:23 +0530 Subject: [PATCH 139/186] Make Tracer injectable (#8696) * Make Tracer injectable Signed-off-by: suranjay * Add changelog entry Signed-off-by: suranjay * Remove entry from CHANGELOG.md Signed-off-by: suranjay --------- Signed-off-by: suranjay --- server/src/main/java/org/opensearch/node/Node.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 24cf822ac6c92..d768165451a5a 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -57,6 +57,7 @@ import org.opensearch.plugins.ExtensionAwarePlugin; import org.opensearch.plugins.SearchPipelinePlugin; import org.opensearch.telemetry.tracing.NoopTracerFactory; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.telemetry.tracing.TracerFactory; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; @@ -378,7 +379,7 @@ public static class DiscoverySettings { private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; - private final TracerFactory tracerFactory; + private final Tracer tracer; final NamedWriteableRegistry namedWriteableRegistry; private final AtomicReference runnableTaskListener; private FileCache fileCache; @@ -1027,6 +1028,7 @@ protected Node( searchModule.getIndexSearcherExecutor(threadPool) ); + TracerFactory tracerFactory; if (FeatureFlags.isEnabled(TELEMETRY)) { final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings()); List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class); @@ -1035,7 +1037,8 @@ protected Node( } else { tracerFactory = new NoopTracerFactory(); } - resourcesToClose.add(tracerFactory::close); + tracer = tracerFactory.getTracer(); + resourcesToClose.add(tracer::close); final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) .stream() @@ -1142,7 +1145,7 @@ protected Node( b.bind(FsHealthService.class).toInstance(fsHealthService); b.bind(SystemIndices.class).toInstance(systemIndices); b.bind(IdentityService.class).toInstance(identityService); - b.bind(TracerFactory.class).toInstance(this.tracerFactory); + b.bind(Tracer.class).toInstance(tracer); }); injector = modules.createInjector(); @@ -1499,7 +1502,7 @@ public synchronized void close() throws IOException { toClose.add(injector.getInstance(NodeEnvironment.class)); toClose.add(stopWatch::stop); if (FeatureFlags.isEnabled(TELEMETRY)) { - toClose.add(injector.getInstance(TracerFactory.class)); + toClose.add(injector.getInstance(Tracer.class)); } if (logger.isTraceEnabled()) { From 27a14c7c811f930b2b36c13f20e1371c6206dd51 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 21:54:35 +0530 Subject: [PATCH 140/186] Add integration tests for remote store restore flow (#8484) Add integration tests for remote store restore flow (#8484) --------- Signed-off-by: Bhumika Saini --- .../remotestore/PrimaryTermValidationIT.java | 2 + .../RemoteStoreBaseIntegTestCase.java | 17 +- .../opensearch/remotestore/RemoteStoreIT.java | 389 +++++++++++++++--- .../cluster/routing/RecoverySource.java | 7 + 4 files changed, 364 insertions(+), 51 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index 6691da81f057d..d1fdde30a9537 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -69,6 +69,8 @@ public void testPrimaryTermValidation() throws Exception { assertAcked( clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) ); + absolutePath2 = randomRepoPath().toAbsolutePath(); + putRepository(absolutePath2, REPOSITORY_2_NAME); // Start data nodes and create index internalCluster().startDataOnlyNodes(2, clusterSettings); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index eb8913591b38b..0b3ceac176193 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -32,10 +32,11 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_NAME = "test-remore-store-repo"; + protected static final String REPOSITORY_2_NAME = "test-remore-store-repo-2"; protected static final int SHARD_COUNT = 1; protected static final int REPLICA_COUNT = 1; - protected Path absolutePath; + protected Path absolutePath2; @Override protected boolean addMockInternalEngine() { @@ -95,10 +96,11 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel } protected Settings remoteTranslogIndexSettings(int numberOfReplicas, int numberOfShards) { + boolean sameRepoForRSSAndRTS = randomBoolean(); return Settings.builder() .put(remoteStoreIndexSettings(numberOfReplicas, numberOfShards)) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, sameRepoForRSSAndRTS ? REPOSITORY_NAME : REPOSITORY_2_NAME) .build(); } @@ -107,20 +109,25 @@ protected Settings remoteTranslogIndexSettings(int numberOfReplicas) { } protected void putRepository(Path path) { - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", path)) - ); + putRepository(path, REPOSITORY_NAME); + } + + protected void putRepository(Path path, String repoName) { + assertAcked(clusterAdmin().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", path))); } protected void setupRepo() { internalCluster().startClusterManagerOnlyNode(); absolutePath = randomRepoPath().toAbsolutePath(); putRepository(absolutePath); + absolutePath2 = randomRepoPath().toAbsolutePath(); + putRepository(absolutePath2, REPOSITORY_2_NAME); } @After public void teardown() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } public int getFileCount(Path path) throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index f01e4969b1fe7..4fcaeabae05fe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -15,6 +15,7 @@ import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.UUIDs; @@ -45,6 +46,8 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; + private static final String INDEX_NAMES = "test-remote-store-1,test-remote-store-2,remote-store-test-index-1,remote-store-test-index-2"; + private static final String INDEX_NAMES_WILDCARD = "test-remote-store-*,remote-store-test-index-*"; private static final String TOTAL_OPERATIONS = "total-operations"; private static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; private static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; @@ -72,27 +75,32 @@ private IndexResponse indexSingleDoc() { .get(); } - private Map indexData(int numberOfIterations, boolean invokeFlush) { + private Map indexData(int numberOfIterations, boolean invokeFlush, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; long maxSeqNo = -1; long maxSeqNoRefreshedOrFlushed = -1; + int shardId = 0; + Map indexingStats = new HashMap<>(); for (int i = 0; i < numberOfIterations; i++) { if (invokeFlush) { - flush(INDEX_NAME); + flush(index); } else { - refresh(INDEX_NAME); + refresh(index); } maxSeqNoRefreshedOrFlushed = maxSeqNo; + indexingStats.put(MAX_SEQ_NO_REFRESHED_OR_FLUSHED + "-shard-" + shardId, maxSeqNoRefreshedOrFlushed); refreshedOrFlushedOperations = totalOperations; int numberOfOperations = randomIntBetween(20, 50); for (int j = 0; j < numberOfOperations; j++) { - IndexResponse response = indexSingleDoc(); + IndexResponse response = INDEX_NAME.equals(index) ? indexSingleDoc() : indexSingleDoc(index); maxSeqNo = response.getSeqNo(); + shardId = response.getShardId().id(); + indexingStats.put(MAX_SEQ_NO_TOTAL + "-shard-" + shardId, maxSeqNo); } totalOperations += numberOfOperations; } - Map indexingStats = new HashMap<>(); + indexingStats.put(TOTAL_OPERATIONS, totalOperations); indexingStats.put(REFRESHED_OR_FLUSHED_OPERATIONS, refreshedOrFlushedOperations); indexingStats.put(MAX_SEQ_NO_TOTAL, maxSeqNo); @@ -100,76 +108,363 @@ private Map indexData(int numberOfIterations, boolean invokeFlush) return indexingStats; } - private void verifyRestoredData(Map indexStats, boolean checkTotal) { + private void verifyRestoredData(Map indexStats, boolean checkTotal, String indexName) { String statsGranularity = checkTotal ? TOTAL_OPERATIONS : REFRESHED_OR_FLUSHED_OPERATIONS; String maxSeqNoGranularity = checkTotal ? MAX_SEQ_NO_TOTAL : MAX_SEQ_NO_REFRESHED_OR_FLUSHED; - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); - assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(statsGranularity)); - IndexResponse response = indexSingleDoc(); - assertEquals(indexStats.get(maxSeqNoGranularity) + 1, response.getSeqNo()); - refresh(INDEX_NAME); - assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(statsGranularity) + 1); + ensureYellowAndNoInitializingShards(indexName); + ensureGreen(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity)); + IndexResponse response = INDEX_NAME.equals(indexName) ? indexSingleDoc() : indexSingleDoc(indexName); + assertEquals(indexStats.get(maxSeqNoGranularity + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo()); + refresh(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); } - private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { - internalCluster().startDataOnlyNodes(3); - if (remoteTranslog) { - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); - } else { - createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); + private void prepareCluster( + int numClusterManagerNodes, + int numDataOnlyNodes, + boolean remoteTranslogEnabled, + String indices, + int replicaCount, + int shardCount + ) { + internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); + internalCluster().startDataOnlyNodes(numDataOnlyNodes); + for (String index : indices.split(",")) { + if (remoteTranslogEnabled) { + createIndex(index, remoteTranslogIndexSettings(replicaCount, shardCount)); + } else { + createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); + } + + ensureYellowAndNoInitializingShards(index); + ensureGreen(index); } - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); + } - Map indexStats = indexData(numberOfIterations, invokeFlush); + /** + * Helper function to test restoring an index with no replication from remote store. Only primary node is dropped. + * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. + * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. + * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. + * @throws IOException IO Exception. + */ + private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { + prepareCluster(0, 3, remoteTranslog, INDEX_NAME, 0, shardCount); + Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + ensureRed(INDEX_NAME); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + + ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); + } + /** + * Helper function to test restoring an index having replicas from remote store when all the nodes housing the primary/replica drop. + * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. + * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. + * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. + * @throws IOException IO Exception. + */ + private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) + throws IOException { + prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1, shardCount); + Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + ensureRed(INDEX_NAME); + internalCluster().startDataOnlyNodes(2); + + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + assertEquals(0, getNumShards(INDEX_NAME).numReplicas); + verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); + } - if (remoteTranslog) { - verifyRestoredData(indexStats, true); - } else { - verifyRestoredData(indexStats, false); + /** + * Helper function to test restoring multiple indices from remote store when all the nodes housing the primary/replica drop. + * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. + * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. + * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. + * @throws IOException IO Exception. + */ + private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) + throws IOException { + prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1, shardCount); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(numberOfIterations, invokeFlush, index); + indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); } - } - public void testRemoteSegmentStoreRestoreWithNoDataPostCommit() throws IOException { - testRestoreFlow(false, 1, true); - } + for (String index : indices) { + ClusterHealthStatus indexHealth = ensureRed(index); + if (ClusterHealthStatus.RED.equals(indexHealth)) { + continue; + } - public void testRemoteSegmentStoreRestoreWithNoDataPostRefresh() throws IOException { - testRestoreFlow(false, 1, false); - } + if (ClusterHealthStatus.GREEN.equals(indexHealth)) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } - public void testRemoteSegmentStoreRestoreWithRefreshedData() throws IOException { - testRestoreFlow(false, randomIntBetween(2, 5), false); - } + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); - public void testRemoteSegmentStoreRestoreWithCommittedData() throws IOException { - testRestoreFlow(false, randomIntBetween(2, 5), true); + assertAcked(client().admin().indices().prepareClose(indices)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")), PlainActionFuture.newFuture()); + ensureGreen(indices); + for (String index : indices) { + assertEquals(shardCount, getNumShards(index).totalNumShards); + verifyRestoredData(indicesStats.get(index), remoteTranslog, index); + } } + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { - testRestoreFlow(true, 1, true); + testRestoreFlow(true, 1, true, randomIntBetween(1, 5)); } + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException { - testRestoreFlow(true, 1, false); + testRestoreFlow(true, 1, false, randomIntBetween(1, 5)); } + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), false); + testRestoreFlow(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), true); + testRestoreFlow(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + } + + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, 1, true, randomIntBetween(1, 5)); + } + + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, 1, false, randomIntBetween(1, 5)); + } + + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + } + + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store + * for multiple indices matching a wildcard name pattern. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") + public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOException { + testRestoreFlowMultipleIndices(true, 2, true, randomIntBetween(1, 5)); + } + + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * with all remote-enabled red indices considered for the restore by default. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") + public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOException { + int shardCount = randomIntBetween(1, 5); + prepareCluster(1, 3, true, INDEX_NAMES, 1, shardCount); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(2, true, index); + indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(new String[] {}), PlainActionFuture.newFuture()); + ensureGreen(indices); + + for (String index : indices) { + assertEquals(shardCount, getNumShards(index).totalNumShards); + verifyRestoredData(indicesStats.get(index), true, index); + } + } + + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * with only some of the remote-enabled red indices requested for the restore. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOException { + int shardCount = randomIntBetween(1, 5); + prepareCluster(1, 3, true, INDEX_NAMES, 0, shardCount); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(2, true, index); + indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indices[0], indices[1]), PlainActionFuture.newFuture()); + ensureGreen(indices[0], indices[1]); + assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); + verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); + assertEquals(shardCount, getNumShards(indices[1]).totalNumShards); + verifyRestoredData(indicesStats.get(indices[1]), true, indices[1]); + ensureRed(indices[2], indices[3]); + } + + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * with all remote-enabled red indices being considered for the restore + * except those matching the specified exclusion pattern. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") + public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOException { + int shardCount = randomIntBetween(1, 5); + prepareCluster(1, 3, true, INDEX_NAMES, 1, shardCount); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(2, true, index); + indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*"), PlainActionFuture.newFuture()); + ensureGreen(indices[0], indices[1]); + assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); + verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); + assertEquals(shardCount, getNumShards(indices[1]).totalNumShards); + verifyRestoredData(indicesStats.get(indices[1]), true, indices[1]); + ensureRed(indices[2], indices[3]); + } + + /** + * Simulates no-op restore from remote store, + * when the index has no data. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + public void testRTSRestoreNoData() throws IOException { + testRestoreFlow(true, 0, true, randomIntBetween(1, 5)); + } + + // TODO: Restore flow - index aliases + private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startDataOnlyNodes(3); if (remoteTranslog) { @@ -180,7 +475,7 @@ private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, bo ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - Map indexStats = indexData(numberOfIterations, invokeFlush); + Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); client().admin() .indices() @@ -263,7 +558,7 @@ private void verifyRemoteStoreCleanup(boolean remoteTranslog) throws Exception { createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); } - indexData(5, randomBoolean()); + indexData(5, randomBoolean(), INDEX_NAME); String indexUUID = client().admin() .indices() .prepareGetSettings(INDEX_NAME) @@ -288,11 +583,12 @@ public void testRemoteTranslogCleanup() throws Exception { verifyRemoteStoreCleanup(true); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8658") public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); int numberOfIterations = randomIntBetween(5, 15); - indexData(numberOfIterations, true); + indexData(numberOfIterations, true, INDEX_NAME); String indexUUID = client().admin() .indices() .prepareGetSettings(INDEX_NAME) @@ -312,11 +608,12 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { }, 30, TimeUnit.SECONDS); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8658") public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); int numberOfIterations = randomIntBetween(5, 15); - indexData(numberOfIterations, false); + indexData(numberOfIterations, false, INDEX_NAME); String indexUUID = client().admin() .indices() .prepareGetSettings(INDEX_NAME) diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index aafc37fba9e85..5cef46689ffc7 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -490,6 +490,13 @@ public int hashCode() { return Objects.hash(restoreUUID, index, version); } + // TODO: This override should be removed/be updated to return "true", + // i.e. we expect no retention leases, once the following issue is fixed: + // https://github.com/opensearch-project/OpenSearch/issues/8795 + @Override + public boolean expectEmptyRetentionLeases() { + return false; + } } /** From 6cc8da561512890c437ef5c605ad99b575ff858b Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Thu, 20 Jul 2023 11:46:37 -0700 Subject: [PATCH 141/186] Change InternalSignificantTerms to only sum shard level counts in final reduce (#8735) Signed-off-by: Jay Deng --- CHANGELOG.md | 1 + .../SignificantTermsSignificanceScoreIT.java | 30 +++++++++++++++ .../search/DefaultSearchContext.java | 6 ++- .../AggregationCollectorManager.java | 2 +- .../aggregations/InternalAggregation.java | 11 ++++++ .../terms/InternalSignificantTerms.java | 16 +++++++- .../internal/FilteredSearchContext.java | 4 +- .../search/internal/SearchContext.java | 2 +- .../SharedSignificantTermsTestMethods.java | 37 +++++++++++++++++++ .../opensearch/test/TestSearchContext.java | 2 +- 10 files changed, 102 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3eda3f852a24..dceec5c62d94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) - Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) +- Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index 33a1f3a4b974a..43d49dc0bfd60 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -31,6 +31,7 @@ package org.opensearch.search.aggregations.bucket; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; @@ -42,6 +43,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.script.MockScriptPlugin; @@ -210,6 +212,34 @@ public void testXContentResponse() throws Exception { } + public void testConsistencyWithDifferentShardCounts() throws Exception { + // The purpose of this test is to validate that the aggregation results do not change with shard count. + // bg_count for significant term agg is summed up across shards, so in this test we compare a 1 shard and 2 shard search request + String type = randomBoolean() ? "text" : "long"; + String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; + SharedSignificantTermsTestMethods.index01Docs(type, settings, this); + + SearchRequestBuilder request = client().prepareSearch(INDEX_NAME) + .setQuery(new TermQueryBuilder(CLASS_FIELD, "0")) + .addAggregation((significantTerms("sig_terms").field(TEXT_FIELD))); + + SearchResponse response1 = request.get(); + + assertAcked(client().admin().indices().delete(new DeleteIndexRequest("*")).get()); + + settings = "{\"index.number_of_shards\": 2, \"index.number_of_replicas\": 0}"; + // We use a custom routing strategy here to ensure that each shard will have at least 1 bucket. + // If there are no buckets collected for a shard, then that will affect the scoring and bg_count and our assertion will not be + // valid. + SharedSignificantTermsTestMethods.index01DocsWithRouting(type, settings, this); + SearchResponse response2 = request.get(); + + assertEquals( + response1.getAggregations().asMap().get("sig_terms").toString(), + response2.getAggregations().asMap().get("sig_terms").toString() + ); + } + public void testPopularTermManyDeletedDocs() throws Exception { String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; assertAcked( diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index ee29d6bfe2b62..f377a5e315e1b 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -917,8 +917,10 @@ public ReaderContext readerContext() { } @Override - public InternalAggregation.ReduceContext partial() { - return requestToAggReduceContextBuilder.apply(request.source()).forPartialReduction(); + public InternalAggregation.ReduceContext partialOnShard() { + InternalAggregation.ReduceContext rc = requestToAggReduceContextBuilder.apply(request.source()).forPartialReduction(); + rc.setSliceLevel(isConcurrentSegmentSearchEnabled()); + return rc; } @Override diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java index 0b36fc8b0cc5a..1f60ff6503ca8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java @@ -70,7 +70,7 @@ public ReduceableSearchResult reduce(Collection collectors) throws IO // using reduce is fine here instead of topLevelReduce as pipeline aggregation is evaluated on the coordinator after all // documents are collected across shards for an aggregation return new AggregationReduceableSearchResult( - InternalAggregations.reduce(Collections.singletonList(internalAggregations), context.partial()) + InternalAggregations.reduce(Collections.singletonList(internalAggregations), context.partialOnShard()) ); } else { return new AggregationReduceableSearchResult(internalAggregations); diff --git a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java index b7577fb647be5..c6d86316fa230 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java +++ b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregation.java @@ -89,6 +89,8 @@ public static class ReduceContext { private final ScriptService scriptService; private final IntConsumer multiBucketConsumer; private final PipelineTree pipelineTreeRoot; + + private boolean isSliceLevel; /** * Supplies the pipelines when the result of the reduce is serialized * to node versions that need pipeline aggregators to be serialized @@ -138,6 +140,7 @@ private ReduceContext( this.multiBucketConsumer = multiBucketConsumer; this.pipelineTreeRoot = pipelineTreeRoot; this.pipelineTreeForBwcSerialization = pipelineTreeForBwcSerialization; + this.isSliceLevel = false; } /** @@ -149,6 +152,14 @@ public boolean isFinalReduce() { return pipelineTreeRoot != null; } + public void setSliceLevel(boolean sliceLevel) { + this.isSliceLevel = sliceLevel; + } + + public boolean isSliceLevel() { + return this.isSliceLevel; + } + public BigArrays bigArrays() { return bigArrays; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java index 6104d2193f6cd..84d148199a7f9 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalSignificantTerms.java @@ -232,7 +232,13 @@ public InternalAggregation reduce(List aggregations, Reduce @SuppressWarnings("unchecked") InternalSignificantTerms terms = (InternalSignificantTerms) aggregation; globalSubsetSize += terms.getSubsetSize(); - globalSupersetSize += terms.getSupersetSize(); + // supersetSize is a shard level count, if we sum it across slices we would produce num_slices_with_bucket * supersetSize where + // num_slices_with_bucket is the number of segment slices that have collected a bucket for the key + if (reduceContext.isSliceLevel()) { + globalSupersetSize = terms.getSupersetSize(); + } else { + globalSupersetSize += terms.getSupersetSize(); + } } Map> buckets = new HashMap<>(); for (InternalAggregation aggregation : aggregations) { @@ -291,7 +297,13 @@ protected B reduceBucket(List buckets, ReduceContext context) { List aggregationsList = new ArrayList<>(buckets.size()); for (B bucket : buckets) { subsetDf += bucket.subsetDf; - supersetDf += bucket.supersetDf; + // supersetDf is a shard level count, if we sum it across slices we would produce num_slices_with_bucket * supersetSize where + // num_slices_with_bucket is the number of segment slices that have collected a bucket for the key + if (context.isSliceLevel()) { + supersetDf = bucket.supersetDf; + } else { + supersetDf += bucket.supersetDf; + } aggregationsList.add(bucket.aggregations); } InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context); diff --git a/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java index 790d2ed5ee4b7..bb990e69e7722 100644 --- a/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java @@ -546,8 +546,8 @@ public ReaderContext readerContext() { } @Override - public InternalAggregation.ReduceContext partial() { - return in.partial(); + public InternalAggregation.ReduceContext partialOnShard() { + return in.partialOnShard(); } @Override diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index fd02ba2ba12bb..c2f81b0d4b8b5 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -465,7 +465,7 @@ public String toString() { public abstract ReaderContext readerContext(); - public abstract InternalAggregation.ReduceContext partial(); + public abstract InternalAggregation.ReduceContext partialOnShard(); // processor used for bucket collectors public abstract void setBucketCollectorProcessor(BucketCollectorProcessor bucketCollectorProcessor); diff --git a/server/src/test/java/org/opensearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/server/src/test/java/org/opensearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index 1092bc4f8f47c..34774758dcd0e 100644 --- a/server/src/test/java/org/opensearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/server/src/test/java/org/opensearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -113,4 +113,41 @@ public static void index01Docs(String type, String settings, OpenSearchIntegTest indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME).setId("7").setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); testCase.indexRandom(true, false, indexRequestBuilderList); } + + public static void index01DocsWithRouting(String type, String settings, OpenSearchIntegTestCase testCase) throws ExecutionException, + InterruptedException { + String textMappings = "type=" + type; + if (type.equals("text")) { + textMappings += ",fielddata=true"; + } + assertAcked( + testCase.prepareCreate(INDEX_NAME) + .setSettings(settings, XContentType.JSON) + .setMapping("text", textMappings, CLASS_FIELD, "type=keyword") + ); + String[] gb = { "0", "1" }; + List indexRequestBuilderList = new ArrayList<>(); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("1").setSource(TEXT_FIELD, "1", CLASS_FIELD, "1").setRouting("0") + ); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("2").setSource(TEXT_FIELD, "1", CLASS_FIELD, "1").setRouting("0") + ); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("3").setSource(TEXT_FIELD, "0", CLASS_FIELD, "0").setRouting("0") + ); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("4").setSource(TEXT_FIELD, "0", CLASS_FIELD, "0").setRouting("1") + ); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("5").setSource(TEXT_FIELD, gb, CLASS_FIELD, "1").setRouting("1") + ); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("6").setSource(TEXT_FIELD, gb, CLASS_FIELD, "0").setRouting("0") + ); + indexRequestBuilderList.add( + client().prepareIndex(INDEX_NAME).setId("7").setSource(TEXT_FIELD, "0", CLASS_FIELD, "0").setRouting("0") + ); + testCase.indexRandom(true, false, indexRequestBuilderList); + } } diff --git a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java index 694f88d944f71..4e44791e77566 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java @@ -659,7 +659,7 @@ public ReaderContext readerContext() { } @Override - public InternalAggregation.ReduceContext partial() { + public InternalAggregation.ReduceContext partialOnShard() { return InternalAggregationTestCase.emptyReduceContextBuilder().forPartialReduction(); } From c47ec1ba79757ff3fdb7ac59c69209c0dcaa3139 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Fri, 21 Jul 2023 11:05:31 +0530 Subject: [PATCH 142/186] Remove remote translog settings to make all remote indices translog enabled (#8719) Remove remote translog setting and restrict to remote only index creation --------- Signed-off-by: bansvaru --- CHANGELOG.md | 2 + distribution/src/config/opensearch.yml | 5 +- ...emoteStoreMockRepositoryIntegTestCase.java | 8 +- ...CreateRemoteIndexClusterDefaultDocRep.java | 27 --- .../remotestore/CreateRemoteIndexIT.java | 179 +++--------------- .../CreateRemoteIndexTranslogDisabledIT.java | 86 --------- .../remotestore/PrimaryTermValidationIT.java | 2 +- .../remotestore/RemoteIndexRecoveryIT.java | 3 +- .../RemoteStoreBaseIntegTestCase.java | 17 +- .../remotestore/RemoteStoreForceMergeIT.java | 2 +- .../opensearch/remotestore/RemoteStoreIT.java | 100 +++------- .../remotestore/RemoteStoreStatsIT.java | 10 +- .../ReplicaToPrimaryPromotionIT.java | 1 - .../SegmentReplicationUsingRemoteStoreIT.java | 4 +- ...tReplicationWithRemoteStorePressureIT.java | 4 +- .../RemoteStoreMultipartFileCorruptionIT.java | 3 +- .../snapshots/RestoreSnapshotIT.java | 56 ++---- .../cluster/metadata/IndexMetadata.java | 40 +--- .../metadata/MetadataCreateIndexService.java | 30 +-- .../common/settings/ClusterSettings.java | 1 - .../common/settings/IndexScopedSettings.java | 1 - .../org/opensearch/index/IndexSettings.java | 8 +- .../opensearch/indices/IndicesService.java | 12 +- .../TransportRemoteStoreStatsActionTests.java | 4 +- .../MetadataCreateIndexServiceTests.java | 56 +----- .../opensearch/index/IndexSettingsTests.java | 110 ++++------- .../index/seqno/ReplicationTrackerTests.java | 23 +-- .../index/shard/IndexShardTests.java | 16 +- .../RemoteStoreRefreshListenerTests.java | 4 + ...overyWithRemoteTranslogOnPrimaryTests.java | 1 - .../SegmentReplicationIndexShardTests.java | 6 +- ...tReplicationWithRemoteIndexShardTests.java | 4 +- ...moteSegmentStoreDirectoryFactoryTests.java | 6 +- .../index/translog/RemoteFSTranslogTests.java | 1 - ...teStorePeerRecoverySourceHandlerTests.java | 1 - .../RemoteStoreReplicationSourceTests.java | 2 + .../blobstore/BlobStoreRepositoryTests.java | 3 +- .../AbstractSnapshotIntegTestCase.java | 3 +- 38 files changed, 203 insertions(+), 638 deletions(-) delete mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexTranslogDisabledIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index dceec5c62d94f..1f809f1f83c88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459)) - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) +- Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) ### Deprecated @@ -63,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018)) - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021)) - Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871)) +- Remove provision to create Remote Indices without Remote Translog Store ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 99e5fb6f9a226..3c4fe822005e0 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -92,10 +92,7 @@ ${path.logs} # cluster.remote_store.enabled: true # # Repository to use for segment upload while enforcing remote store for an index -# cluster.remote_store.repository: my-repo-1 -# -# Controls whether cluster imposes index creation only with translog remote store enabled -# cluster.remote_store.translog.enabled: true +# cluster.remote_store.segment.repository: my-repo-1 # # Repository to use for translog upload while enforcing remote store for an index # cluster.remote_store.translog.repository: my-repo-1 diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index bb8839a47eb69..709c027c3f347 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -34,6 +34,7 @@ public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { protected static final String REPOSITORY_NAME = "my-segment-repo-1"; + protected static final String TRANSLOG_REPOSITORY_NAME = "my-translog-repo-1"; protected static final String INDEX_NAME = "remote-store-test-idx-1"; @Override @@ -62,13 +63,16 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) { .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, TRANSLOG_REPOSITORY_NAME) .build(); } protected void deleteRepo() { logger.info("--> Deleting the repository={}", REPOSITORY_NAME); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + logger.info("--> Deleting the repository={}", TRANSLOG_REPOSITORY_NAME); + assertAcked(clusterAdmin().prepareDeleteRepository(TRANSLOG_REPOSITORY_NAME)); } protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { @@ -88,6 +92,8 @@ protected String setup(Path repoLocation, double ioFailureRate, String skipExcep .put("skip_exception_on_blobs", skipExceptionBlobList) .put("max_failure_number", maxFailure) ); + logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation); + createRepository(TRANSLOG_REPOSITORY_NAME, "mock", Settings.builder().put("location", repoLocation)); String dataNodeName = internalCluster().startDataOnlyNodes(1).get(0); createIndex(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java index 40b608388736a..2abf4fc50ec69 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java @@ -17,7 +17,6 @@ import org.opensearch.test.OpenSearchIntegTestCase; import static org.hamcrest.Matchers.containsString; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -33,31 +32,6 @@ protected Settings nodeSettings(int nodeOriginal) { return builder.build(); } - @Override - public void testRemoteStoreTranslogDisabledByUser() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) - .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-segment-repo-1", - "false", - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL - ); - } - @Override public void testDefaultRemoteStoreNoUserOverride() throws Exception { Settings settings = Settings.builder() @@ -90,7 +64,6 @@ public void testDefaultRemoteStoreNoUserOverrideExceptReplicationTypeSegment() t indexSettings, "true", "my-segment-repo-1", - "true", "my-translog-repo-1", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index fda344acad166..e52a12f66cff4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -21,17 +21,17 @@ import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; +import java.util.Locale; + import static org.hamcrest.Matchers.containsString; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -53,7 +53,6 @@ protected Settings nodeSettings(int nodeOriginal) { .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .put(settings); return builder.build(); @@ -100,7 +99,6 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception { indexSettings, "true", "my-segment-repo-1", - "true", "my-translog-repo-1", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL @@ -124,7 +122,6 @@ public void testRemoteStoreDisabledByUser() throws Exception { "false", null, null, - null, client().settings().get(CLUSTER_SETTING_REPLICATION_TYPE), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); @@ -161,7 +158,13 @@ public void testRemoteStoreEnabledByUserWithoutRemoteRepoIllegalArgumentExceptio ); assertThat( exc.getMessage(), - containsString("Setting index.remote_store.repository should be provided with non-empty repository ID") + containsString( + String.format( + Locale.ROOT, + "Setting %s should be provided with non-empty repository ID", + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + ) + ) ); } @@ -182,7 +185,6 @@ public void testReplicationTypeDocumentByUser() throws Exception { null, null, null, - null, ReplicationType.DOCUMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); @@ -192,7 +194,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -200,7 +202,14 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI ); assertThat( exc.getMessage(), - containsString("Settings index.remote_store.repository can only be set/enabled when index.remote_store.enabled is set to true") + containsString( + String.format( + Locale.ROOT, + "Settings %s can only be set/enabled when %s is set to true", + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + SETTING_REMOTE_STORE_ENABLED + ) + ) ); } @@ -210,7 +219,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") .build(); assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); @@ -223,36 +232,12 @@ public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception { indexSettings, "true", "my-custom-repo", - "true", "my-translog-repo-1", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); } - public void testRemoteStoreTranslogDisabledByUser() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) - .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-segment-repo-1", - "false", - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL - ); - } - public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() throws Exception { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) @@ -266,115 +251,12 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th assertThat( exc.getMessage(), containsString( - "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true" - ) - ); - } - - public void testRemoteStoreOverrideOnlyTranslogEnabled() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .build(); - - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() - ); - assertThat( - exc.getMessage(), - containsString( - "Settings index.remote_store.translog.enabled can only be set/enabled when index.remote_store.enabled is set to true" - ) - ); - } - - public void testRemoteStoreOverrideOnlyTranslogEnabledAndRepo() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") - .build(); - - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() - ); - assertThat( - exc.getMessage(), - containsString( - "Settings index.remote_store.translog.enabled can only be set/enabled when index.remote_store.enabled is set to true" - ) - ); - } - - public void testRemoteStoreOverrideTranslogDisabledCorrectly() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) - .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-custom-repo", - "false", - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL - ); - } - - public void testRemoteStoreOverrideTranslogDisabledWithTranslogRepoIllegalArgumentException() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") - .build(); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() - ); - assertThat( - exc.getMessage(), - containsString( - "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true" - ) - ); - } - - public void testRemoteStoreOverrideOnlyTranslogRepoWithRemoteStoreEnabledIllegalArgumentException() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") - .build(); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() - ); - assertThat( - exc.getMessage(), - containsString( - "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true" + String.format( + Locale.ROOT, + "Settings %s can only be set/enabled when %s is set to true", + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + SETTING_REMOTE_STORE_ENABLED + ) ) ); } @@ -385,8 +267,7 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") - .put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") .build(); assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); @@ -399,7 +280,6 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { indexSettings, "true", "my-custom-repo", - "true", "my-custom-repo", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL @@ -423,7 +303,6 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() throws Excepti null, null, null, - null, ReplicationType.DOCUMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); @@ -433,15 +312,13 @@ protected void verifyRemoteStoreIndexSettings( Settings indexSettings, String isRemoteSegmentEnabled, String remoteSegmentRepo, - String isRemoteTranslogEnabled, String remoteTranslogRepo, String replicationType, TimeValue translogBufferInterval ) { assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE)); assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY)); - assertEquals(isRemoteTranslogEnabled, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_ENABLED)); + assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexTranslogDisabledIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexTranslogDisabledIT.java deleted file mode 100644 index 01c463392d7b3..0000000000000 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexTranslogDisabledIT.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.remotestore; - -import org.opensearch.action.admin.indices.get.GetIndexRequest; -import org.opensearch.action.admin.indices.get.GetIndexResponse; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.IndexSettings; -import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.OpenSearchIntegTestCase; - -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; - -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class CreateRemoteIndexTranslogDisabledIT extends CreateRemoteIndexIT { - - @Override - protected Settings nodeSettings(int nodeOriginal) { - Settings settings = super.nodeSettings(nodeOriginal); - Settings.Builder builder = Settings.builder().put(settings); - builder.remove(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey()); - builder.remove(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey()); - return builder.build(); - } - - public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo") - .build(); - - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-custom-repo", - null, - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL - ); - } - - public void testDefaultRemoteStoreNoUserOverride() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-segment-repo-1", - null, - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL - ); - } - -} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index d1fdde30a9537..9d63c9b528314 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -74,7 +74,7 @@ public void testPrimaryTermValidation() throws Exception { // Start data nodes and create index internalCluster().startDataOnlyNodes(2, clusterSettings); - createIndex(INDEX_NAME, remoteTranslogIndexSettings(1)); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index 11c9993ac7874..4f7961cec22d7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -58,8 +58,7 @@ public Settings indexSettings() { .put(super.indexSettings()) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 0b3ceac176193..c5d023bdd7a64 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -64,11 +64,13 @@ IndexResponse indexSingleDoc(String indexName) { } private Settings defaultIndexSettings() { + boolean sameRepoForRSSAndRTS = randomBoolean(); return Settings.builder() .put(super.indexSettings()) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, sameRepoForRSSAndRTS ? REPOSITORY_NAME : REPOSITORY_2_NAME) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, SHARD_COUNT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") @@ -95,19 +97,6 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel .build(); } - protected Settings remoteTranslogIndexSettings(int numberOfReplicas, int numberOfShards) { - boolean sameRepoForRSSAndRTS = randomBoolean(); - return Settings.builder() - .put(remoteStoreIndexSettings(numberOfReplicas, numberOfShards)) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, sameRepoForRSSAndRTS ? REPOSITORY_NAME : REPOSITORY_2_NAME) - .build(); - } - - protected Settings remoteTranslogIndexSettings(int numberOfReplicas) { - return remoteTranslogIndexSettings(numberOfReplicas, 1); - } - protected void putRepository(Path path) { putRepository(path, REPOSITORY_NAME); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index 1e9458069d459..b4456f887cbaa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -97,7 +97,7 @@ private void verifyRestoredData(Map indexStats, long deletedDocs) private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) throws IOException { - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); + createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 4fcaeabae05fe..42bd4b5173fa3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -120,23 +120,11 @@ private void verifyRestoredData(Map indexStats, boolean checkTotal assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); } - private void prepareCluster( - int numClusterManagerNodes, - int numDataOnlyNodes, - boolean remoteTranslogEnabled, - String indices, - int replicaCount, - int shardCount - ) { + private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); for (String index : indices.split(",")) { - if (remoteTranslogEnabled) { - createIndex(index, remoteTranslogIndexSettings(replicaCount, shardCount)); - } else { - createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); - } - + createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); ensureYellowAndNoInitializingShards(index); ensureGreen(index); } @@ -144,13 +132,12 @@ private void prepareCluster( /** * Helper function to test restoring an index with no replication from remote store. Only primary node is dropped. - * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { - prepareCluster(0, 3, remoteTranslog, INDEX_NAME, 0, shardCount); + private void testRestoreFlow(int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { + prepareCluster(0, 3, INDEX_NAME, 0, shardCount); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -162,7 +149,7 @@ private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boo ensureGreen(INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); - verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); + verifyRestoredData(indexStats, true, INDEX_NAME); } /** @@ -174,7 +161,7 @@ private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boo */ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { - prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1, shardCount); + prepareCluster(1, 2, INDEX_NAME, 1, shardCount); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -187,21 +174,20 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); assertEquals(0, getNumShards(INDEX_NAME).numReplicas); - verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); + verifyRestoredData(indexStats, true, INDEX_NAME); } /** * Helper function to test restoring multiple indices from remote store when all the nodes housing the primary/replica drop. - * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) - throws IOException { - prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1, shardCount); + private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { + prepareCluster(1, 3, INDEX_NAMES, 1, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -233,7 +219,7 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf ensureGreen(indices); for (String index : indices) { assertEquals(shardCount, getNumShards(index).totalNumShards); - verifyRestoredData(indicesStats.get(index), remoteTranslog, index); + verifyRestoredData(indicesStats.get(index), true, index); } } @@ -243,7 +229,7 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { - testRestoreFlow(true, 1, true, randomIntBetween(1, 5)); + testRestoreFlow(1, true, randomIntBetween(1, 5)); } /** @@ -251,7 +237,7 @@ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException { - testRestoreFlow(true, 1, false, randomIntBetween(1, 5)); + testRestoreFlow(1, false, randomIntBetween(1, 5)); } /** @@ -260,7 +246,7 @@ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); + testRestoreFlow(randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } /** @@ -269,7 +255,7 @@ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + testRestoreFlow(randomIntBetween(2, 5), true, randomIntBetween(1, 5)); } /** @@ -319,7 +305,7 @@ public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOExcepti */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOException { - testRestoreFlowMultipleIndices(true, 2, true, randomIntBetween(1, 5)); + testRestoreFlowMultipleIndices(2, true, randomIntBetween(1, 5)); } /** @@ -331,7 +317,7 @@ public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOEx @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOException { int shardCount = randomIntBetween(1, 5); - prepareCluster(1, 3, true, INDEX_NAMES, 1, shardCount); + prepareCluster(1, 3, INDEX_NAMES, 1, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -375,7 +361,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio */ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOException { int shardCount = randomIntBetween(1, 5); - prepareCluster(1, 3, true, INDEX_NAMES, 0, shardCount); + prepareCluster(1, 3, INDEX_NAMES, 0, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -417,7 +403,7 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOException { int shardCount = randomIntBetween(1, 5); - prepareCluster(1, 3, true, INDEX_NAMES, 1, shardCount); + prepareCluster(1, 3, INDEX_NAMES, 1, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -460,18 +446,14 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") public void testRTSRestoreNoData() throws IOException { - testRestoreFlow(true, 0, true, randomIntBetween(1, 5)); + testRestoreFlow(0, true, randomIntBetween(1, 5)); } // TODO: Restore flow - index aliases - private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws Exception { + private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startDataOnlyNodes(3); - if (remoteTranslog) { - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); - } else { - createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); - } + createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -518,45 +500,25 @@ private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, bo ); } - public void testPeerRecoveryWithRemoteStoreNoRemoteTranslogNoDataFlush() throws Exception { - testPeerRecovery(false, 1, true); - } - - public void testPeerRecoveryWithRemoteStoreNoRemoteTranslogFlush() throws Exception { - testPeerRecovery(false, randomIntBetween(2, 5), true); - } - - public void testPeerRecoveryWithRemoteStoreNoRemoteTranslogNoDataRefresh() throws Exception { - testPeerRecovery(false, 1, false); - } - - public void testPeerRecoveryWithRemoteStoreNoRemoteTranslogRefresh() throws Exception { - testPeerRecovery(false, randomIntBetween(2, 5), false); - } - public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataFlush() throws Exception { - testPeerRecovery(true, 1, true); + testPeerRecovery(1, true); } public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogFlush() throws Exception { - testPeerRecovery(true, randomIntBetween(2, 5), true); + testPeerRecovery(randomIntBetween(2, 5), true); } public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataRefresh() throws Exception { - testPeerRecovery(true, 1, false); + testPeerRecovery(1, false); } public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh() throws Exception { - testPeerRecovery(true, randomIntBetween(2, 5), false); + testPeerRecovery(randomIntBetween(2, 5), false); } - private void verifyRemoteStoreCleanup(boolean remoteTranslog) throws Exception { + private void verifyRemoteStoreCleanup() throws Exception { internalCluster().startDataOnlyNodes(3); - if (remoteTranslog) { - createIndex(INDEX_NAME, remoteTranslogIndexSettings(1)); - } else { - createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); - } + createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); indexData(5, randomBoolean(), INDEX_NAME); String indexUUID = client().admin() @@ -575,12 +537,8 @@ private void verifyRemoteStoreCleanup(boolean remoteTranslog) throws Exception { }, 30, TimeUnit.SECONDS); } - public void testRemoteSegmentCleanup() throws Exception { - verifyRemoteStoreCleanup(false); - } - public void testRemoteTranslogCleanup() throws Exception { - verifyRemoteStoreCleanup(true); + verifyRemoteStoreCleanup(); } @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8658") diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 28b01e0ab1ef6..76ef153fab963 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -36,11 +36,7 @@ public void testStatsResponseFromAllNodes() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. - if (randomBoolean()) { - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); - } else { - createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); - } + createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -69,7 +65,7 @@ public void testStatsResponseAllShards() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0, 3)); + createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 3)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -93,7 +89,7 @@ public void testStatsResponseFromLocalNode() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0, 3)); + createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 3)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index 0f3e041dd429a..6764c50175e61 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -42,7 +42,6 @@ public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shard_count) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index d39b30ada5ef7..f298fac7c894e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -36,8 +36,8 @@ public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java index e7c9c811b7734..0b64680033d84 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java @@ -36,8 +36,8 @@ public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java index 8f375ca6e2b01..529e84d281476 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java @@ -79,7 +79,8 @@ protected Settings remoteStoreIndexSettings() { .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index 5649169ee15b7..dbd96a7fd109f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -87,7 +87,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.index.query.QueryBuilders.matchQuery; @@ -175,15 +175,10 @@ public void testParallelRestoreOperations() { } public void testRestoreRemoteStoreIndicesWithRemoteTranslog() throws IOException, ExecutionException, InterruptedException { - testRestoreOperationsShallowCopyEnabled(true); + testRestoreOperationsShallowCopyEnabled(); } - public void testRestoreRemoteStoreIndicesWithoutRemoteTranslog() throws IOException, ExecutionException, InterruptedException { - testRestoreOperationsShallowCopyEnabled(false); - } - - public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnabled) throws IOException, ExecutionException, - InterruptedException { + public void testRestoreOperationsShallowCopyEnabled() throws IOException, ExecutionException, InterruptedException { String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); String primary = internalCluster().startDataOnlyNode(); String indexName1 = "testindex1"; @@ -205,10 +200,10 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable createRepository(remoteStoreRepoName, "fs", absolutePath2); Client client = client(); - Settings indexSettings = getIndexSettings(true, remoteTranslogEnabled, remoteStoreRepoName, 1, 0).build(); + Settings indexSettings = getIndexSettings(true, remoteStoreRepoName, 1, 0).build(); createIndex(indexName1, indexSettings); - Settings indexSettings2 = getIndexSettings(false, false, null, 1, 0).build(); + Settings indexSettings2 = getIndexSettings(false, null, 1, 0).build(); createIndex(indexName2, indexSettings2); final int numDocsInIndex1 = 5; @@ -295,7 +290,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable .cluster() .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) .setWaitForCompletion(false) - .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED) + .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED) .setIndices(indexName1) .setRenamePattern(indexName1) .setRenameReplacement(restoredIndexName1Seg) @@ -309,7 +304,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable .get(); indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg); assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertNull(indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY, null)); + assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1); // indexing some new docs and validating @@ -322,11 +317,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable .cluster() .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) .setWaitForCompletion(false) - .setIgnoreIndexSettings( - IndexMetadata.SETTING_REMOTE_STORE_ENABLED, - IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, - IndexMetadata.SETTING_REPLICATION_TYPE - ) + .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REPLICATION_TYPE) .setIndices(indexName1) .setRenamePattern(indexName1) .setRenameReplacement(restoredIndexName1Doc) @@ -340,7 +331,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable .get(); indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc); assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertNull(indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY, null)); + assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1); // indexing some new docs and validating @@ -369,10 +360,10 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { createRepository(remoteStoreRepoName, "fs", absolutePath2); Client client = client(); - Settings indexSettings = getIndexSettings(true, randomBoolean(), remoteStoreRepoName, 1, 0).build(); + Settings indexSettings = getIndexSettings(true, remoteStoreRepoName, 1, 0).build(); createIndex(indexName1, indexSettings); - Settings indexSettings2 = getIndexSettings(false, false, null, 1, 0).build(); + Settings indexSettings2 = getIndexSettings(false, null, 1, 0).build(); createIndex(indexName2, indexSettings2); final int numDocsInIndex1 = 5; @@ -472,10 +463,10 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException createRepository(remoteStoreRepo2Name, "fs", absolutePath3); Client client = client(); - Settings indexSettings = getIndexSettings(true, true, remoteStoreRepoName, 1, 0).build(); + Settings indexSettings = getIndexSettings(true, remoteStoreRepoName, 1, 0).build(); createIndex(indexName1, indexSettings); - Settings indexSettings2 = getIndexSettings(false, false, null, 1, 0).build(); + Settings indexSettings2 = getIndexSettings(false, null, 1, 0).build(); createIndex(indexName2, indexSettings2); final int numDocsInIndex1 = 5; @@ -501,7 +492,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); Settings remoteStoreIndexSettings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo2Name) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name) .build(); // restore index as a remote store index with different remote store repo RestoreSnapshotResponse restoreSnapshotResponse = client.admin() @@ -534,28 +525,18 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); } - private Settings.Builder getIndexSettings( - boolean enableRemoteStore, - boolean enableRemoteTranslog, - String remoteStoreRepo, - int numOfShards, - int numOfReplicas - ) { + private Settings.Builder getIndexSettings(boolean enableRemoteStore, String remoteStoreRepo, int numOfShards, int numOfReplicas) { Settings.Builder settingsBuilder = Settings.builder() .put(super.indexSettings()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas); if (enableRemoteStore) { settingsBuilder.put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); } - if (enableRemoteTranslog) { - settingsBuilder.put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) - .build(); - } return settingsBuilder; } @@ -588,7 +569,8 @@ public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionExce Settings indexSettings = Settings.builder() .put(super.indexSettings()) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepoName) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepoName) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepoName) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index c94f99150bd5e..1ba38daa40566 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -291,9 +291,7 @@ public Iterator> settings() { public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled"; - public static final String SETTING_REMOTE_STORE_REPOSITORY = "index.remote_store.repository"; - - public static final String SETTING_REMOTE_TRANSLOG_STORE_ENABLED = "index.remote_store.translog.enabled"; + public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository"; public static final String SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY = "index.remote_store.translog.repository"; @@ -337,7 +335,7 @@ public Iterator> settings() { * Used to specify remote store repository to use for this index. */ public static final Setting INDEX_REMOTE_STORE_REPOSITORY_SETTING = Setting.simpleString( - SETTING_REMOTE_STORE_REPOSITORY, + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, new Setting.Validator<>() { @Override @@ -377,34 +375,6 @@ private static void validateRemoteStoreSettingEnabled(final Map, Obje } } - /** - * Used to specify if the index translog operations should be persisted in the remote store. - */ - public static final Setting INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING = Setting.boolSetting( - SETTING_REMOTE_TRANSLOG_STORE_ENABLED, - false, - new Setting.Validator<>() { - - @Override - public void validate(final Boolean value) {} - - @Override - public void validate(final Boolean value, final Map, Object> settings) { - if (value == true) { - validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING); - } - } - - @Override - public Iterator> settings() { - final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); - return settings.iterator(); - } - }, - Property.IndexScope, - Property.Final - ); - public static final Setting INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString( SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, new Setting.Validator<>() { @@ -419,13 +389,13 @@ public void validate(final String value, final Map, Object> settings) "Setting " + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID" ); } else { - final Boolean isRemoteTranslogStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING); + final Boolean isRemoteTranslogStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); if (isRemoteTranslogStoreEnabled == null || isRemoteTranslogStoreEnabled == false) { throw new IllegalArgumentException( "Settings " + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() + " can only be set/enabled when " - + INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey() + + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + " is set to true" ); } @@ -434,7 +404,7 @@ public void validate(final String value, final Map, Object> settings) @Override public Iterator> settings() { - final List> settings = Collections.singletonList(INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING); + final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); return settings.iterator(); } }, diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 97798f9627c60..728bac647d74a 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -123,7 +123,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REPLICATION_TYPE_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; @@ -131,15 +130,13 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; /** @@ -973,29 +970,20 @@ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, } settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true); - String remoteStoreRepo; if (Objects.equals(requestSettings.get(INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()), "true")) { remoteStoreRepo = requestSettings.get(INDEX_REMOTE_STORE_REPOSITORY_SETTING.getKey()); } else { remoteStoreRepo = CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.get(clusterSettings); } - settingsBuilder.put(SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo); - - boolean translogStoreEnabled = false; - String translogStoreRepo = null; - if (Objects.equals(requestSettings.get(INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey()), "true")) { - translogStoreEnabled = true; - translogStoreRepo = requestSettings.get(INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey()); - } else if (Objects.equals(requestSettings.get(INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey()), "false") == false - && CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.get(clusterSettings)) { - translogStoreEnabled = true; - translogStoreRepo = CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings); - } - if (translogStoreEnabled) { - settingsBuilder.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, translogStoreEnabled) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogStoreRepo); - } + settingsBuilder.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) + .put( + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + requestSettings.get( + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), + CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings) + ) + ); } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index d70ea16cf5fdd..46a43842451d9 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -672,7 +672,6 @@ public void apply(Settings value, Settings current, Settings previous) { List.of( IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING, IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING, - IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING, IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING ), List.of(FeatureFlags.CONCURRENT_SEGMENT_SEARCH), diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 739982036c2af..3cc7c351fe1bf 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -236,7 +236,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { List.of( IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING, - IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING, IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING ), FeatureFlags.CONCURRENT_SEGMENT_SEARCH, diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 7a369c8edcb2e..1fb036580af27 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -612,7 +612,6 @@ public final class IndexSettings { private final int numberOfShards; private final ReplicationType replicationType; private final boolean isRemoteStoreEnabled; - private final boolean isRemoteTranslogStoreEnabled; private volatile TimeValue remoteTranslogUploadBufferInterval; private final String remoteStoreTranslogRepository; private final String remoteStoreRepository; @@ -782,10 +781,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); - isRemoteTranslogStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false); remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY); remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings); - remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY); + remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY); isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(this.settings); if (isRemoteSnapshot && FeatureFlags.isEnabled(SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY)) { @@ -1070,7 +1068,9 @@ public boolean isRemoteStoreEnabled() { * Returns if remote translog store is enabled for this index. */ public boolean isRemoteTranslogStoreEnabled() { - return isRemoteTranslogStoreEnabled; + // Today enabling remote store automatically enables remote translog as well. + // which is why isRemoteStoreEnabled is used to represent isRemoteTranslogStoreEnabled + return isRemoteStoreEnabled; } /** diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 0ffad8ce65b7a..b574ffd1006c0 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -257,22 +257,12 @@ public class IndicesService extends AbstractLifecycleComponent * Used to specify default repo to use for segment upload for remote store backed indices */ public static final Setting CLUSTER_REMOTE_STORE_REPOSITORY_SETTING = Setting.simpleString( - "cluster.remote_store.repository", + "cluster.remote_store.segment.repository", "", Property.NodeScope, Property.Final ); - /** - * Used to specify if all indexes are to create with translog remote store enabled by default - */ - public static final Setting CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING = Setting.boolSetting( - "cluster.remote_store.translog.enabled", - false, - Property.NodeScope, - Property.Final - ); - /** * Used to specify default repo to use for translog upload for remote store backed indices */ diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index f3f6a570c31ba..25e44884814a5 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -46,7 +46,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; public class TransportRemoteStoreStatsActionTests extends IndexShardTestCase { private IndicesService indicesService; @@ -71,7 +71,7 @@ public void setUp() throws Exception { .settings( settings(Version.CURRENT).put(SETTING_INDEX_UUID, INDEX.getUUID()) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-test-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-test-repo") .build() ) .numberOfShards(2) diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index da02e6dca680e..e52237c8dba99 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -131,8 +131,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; @@ -147,7 +146,6 @@ import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; @@ -1198,7 +1196,6 @@ public void testRemoteStoreNoUserOverrideConflictingReplicationTypeIndexSettings .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1228,7 +1225,6 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1251,7 +1247,6 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin indexSettings, "true", "my-segment-repo-1", - "true", "my-translog-repo-1", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL @@ -1263,7 +1258,6 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1283,7 +1277,6 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { indexSettings, "true", "my-segment-repo-1", - "true", "my-translog-repo-1", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL @@ -1318,41 +1311,6 @@ public void testRemoteStoreDisabledByUserIndexSettings() { "false", null, null, - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL - ); - } - - public void testRemoteStoreTranslogDisabledByUserIndexSettings() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder(); - requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false); - request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-segment-repo-1", - "false", - null, ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); @@ -1363,7 +1321,6 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1372,7 +1329,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo"); + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo"); request.settings(requestSettings.build()); Settings indexSettings = aggregateIndexSettings( ClusterState.EMPTY_STATE, @@ -1388,7 +1345,6 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { indexSettings, "true", "my-custom-repo", - "true", "my-translog-repo-1", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL @@ -1406,7 +1362,7 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); final Settings.Builder requestSettings = Settings.builder(); - requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo").put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true); + requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo"); request.settings(requestSettings.build()); Settings indexSettings = aggregateIndexSettings( ClusterState.EMPTY_STATE, @@ -1422,7 +1378,6 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { indexSettings, "true", "my-segment-repo-1", - "true", "my-custom-repo", ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL @@ -1457,7 +1412,6 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() { null, null, null, - null, ReplicationType.DOCUMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); @@ -1702,15 +1656,13 @@ private void verifyRemoteStoreIndexSettings( Settings indexSettings, String isRemoteSegmentEnabled, String remoteSegmentRepo, - String isRemoteTranslogEnabled, String remoteTranslogRepo, String replicationType, TimeValue translogBufferInterval ) { assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE)); assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY)); - assertEquals(isRemoteTranslogEnabled, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_ENABLED)); + assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)); } diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 26e53ebc00b26..e42e9b4970081 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -54,6 +54,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Locale; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -794,30 +795,6 @@ public void testRemoteTranslogStoreDefaultSetting() { assertFalse(settings.isRemoteTranslogStoreEnabled()); } - public void testRemoteTranslogStoreExplicitSetting() { - IndexMetadata metadata = newIndexMeta( - "index", - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .build() - ); - IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertTrue(settings.isRemoteTranslogStoreEnabled()); - } - - public void testRemoteTranslogStoreNullSetting() { - Settings indexSettings = Settings.builder() - .put("index.remote_store.translog.enabled", "null") - .put("index.remote_store.enabled", randomBoolean()) - .build(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.get(indexSettings) - ); - assertEquals("Failed to parse value [null] as only [true] or [false] are allowed.", iae.getMessage()); - } - public void testUpdateRemoteStoreFails() { Set> remoteStoreSettingSet = new HashSet<>(); remoteStoreSettingSet.add(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING); @@ -834,37 +811,6 @@ public void testUpdateRemoteStoreFails() { assertEquals(error.getMessage(), "final index setting [index.remote_store.enabled], not updateable"); } - public void testUpdateRemoteTranslogStoreFails() { - Set> remoteStoreSettingSet = new HashSet<>(); - remoteStoreSettingSet.add(IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING); - IndexScopedSettings settings = new IndexScopedSettings(Settings.EMPTY, remoteStoreSettingSet); - SettingsException error = expectThrows( - SettingsException.class, - () -> settings.updateSettings( - Settings.builder().put("index.remote_store.translog.enabled", randomBoolean()).build(), - Settings.builder(), - Settings.builder(), - "index" - ) - ); - assertEquals(error.getMessage(), "final index setting [index.remote_store.translog.enabled], not updateable"); - } - - public void testEnablingRemoteTranslogStoreFailsWhenRemoteSegmentDisabled() { - Settings indexSettings = Settings.builder() - .put("index.remote_store.translog.enabled", true) - .put("index.remote_store.enabled", false) - .build(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.get(indexSettings) - ); - assertEquals( - "Settings index.remote_store.translog.enabled can only be set/enabled when index.remote_store.enabled is set to true", - iae.getMessage() - ); - } - public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDocument() { Settings indexSettings = Settings.builder() .put("index.replication.type", ReplicationType.DOCUMENT) @@ -901,7 +847,7 @@ public void testRemoteRepositoryExplicitSetting() { Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); @@ -915,42 +861,57 @@ public void testUpdateRemoteRepositoryFails() { SettingsException error = expectThrows( SettingsException.class, () -> settings.updateSettings( - Settings.builder().put("index.remote_store.repository", randomUnicodeOfLength(10)).build(), + Settings.builder().put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, randomUnicodeOfLength(10)).build(), Settings.builder(), Settings.builder(), "index" ) ); - assertEquals(error.getMessage(), "final index setting [index.remote_store.repository], not updateable"); + assertEquals( + error.getMessage(), + String.format(Locale.ROOT, "final index setting [%s], not updateable", IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY) + ); } public void testSetRemoteRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() - .put("index.replication.type", ReplicationType.SEGMENT) - .put("index.remote_store.enabled", false) - .put("index.remote_store.repository", "repo1") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( - "Settings index.remote_store.repository can only be set/enabled when index.remote_store.enabled is set to true", + String.format( + Locale.ROOT, + "Settings %s can only be set/enabled when %s is set to true", + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + IndexMetadata.SETTING_REMOTE_STORE_ENABLED + ), iae.getMessage() ); } public void testSetRemoteRepositoryFailsWhenEmptyString() { Settings indexSettings = Settings.builder() - .put("index.replication.type", ReplicationType.SEGMENT) - .put("index.remote_store.enabled", false) - .put("index.remote_store.repository", "") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING.get(indexSettings) ); - assertEquals("Setting index.remote_store.repository should be provided with non-empty repository ID", iae.getMessage()); + assertEquals( + String.format( + Locale.ROOT, + "Setting %s should be provided with non-empty repository ID", + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + ), + iae.getMessage() + ); } public void testRemoteTranslogRepoDefaultSetting() { @@ -967,7 +928,6 @@ public void testRemoteTranslogExplicitSetting() { "index", Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "tlog-store") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "200ms") .build() @@ -978,10 +938,10 @@ public void testRemoteTranslogExplicitSetting() { assertEquals(TimeValue.timeValueMillis(200), settings.getRemoteTranslogUploadBufferInterval()); } - public void testSetRemoteTranslogRepositoryFailsWhenRemoteTranslogIsNotEnabled() { + public void testSetRemoteTranslogRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() - .put("index.replication.type", ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "repo1") .build(); IllegalArgumentException iae = expectThrows( @@ -989,15 +949,15 @@ public void testSetRemoteTranslogRepositoryFailsWhenRemoteTranslogIsNotEnabled() () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( - "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true", + "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.enabled is set to true", iae.getMessage() ); } public void testSetRemoteTranslogRepositoryFailsWhenEmptyString() { Settings indexSettings = Settings.builder() - .put("index.replication.type", ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "") .build(); IllegalArgumentException iae = expectThrows( @@ -1011,7 +971,7 @@ public void testSetRemoteTranslogBufferIntervalDefaultSetting() { Version createdVersion = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); Settings settings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .build(); assertEquals(TimeValue.timeValueMillis(650), IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings)); } @@ -1019,7 +979,7 @@ public void testSetRemoteTranslogBufferIntervalDefaultSetting() { public void testSetRemoteTranslogBufferIntervalFailsWhenEmpty() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "") .build(); IllegalArgumentException iae = expectThrows( diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java index 822cae1b9392d..d56cbda2fe6e4 100644 --- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.codecs.Codec; import org.opensearch.action.ActionListener; import org.opensearch.action.support.replication.ReplicationResponse; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; @@ -1292,7 +1293,7 @@ public void testGlobalCheckpointUpdateWithRemoteTranslogEnabled() { assertThat(allocations.size(), equalTo(active.size() + initializing.size())); final AllocationId primaryId = active.iterator().next(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO)); @@ -1367,7 +1368,7 @@ public void testUpdateFromClusterManagerWithRemoteTranslogEnabled() { assertThat(allocations.size(), equalTo(active.size() + initializing.size())); final AllocationId primaryId = active.iterator().next(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO)); @@ -1437,7 +1438,7 @@ public void testUpdateFromClusterManagerWithRemoteTranslogEnabled() { */ public void testUpdateGlobalCheckpointOnReplicaWithRemoteTranslogEnabled() { final AllocationId active = AllocationId.newInitializing(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(active, settings); final long globalCheckpoint = randomLongBetween(NO_OPS_PERFORMED, Long.MAX_VALUE - 1); tracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "test"); @@ -1459,7 +1460,7 @@ public void testMarkAllocationIdAsInSyncWithRemoteTranslogEnabled() throws Excep Set initializing = new HashSet<>(initializingWithCheckpoints.keySet()); final AllocationId primaryId = active.iterator().next(); final AllocationId replicaId = initializing.iterator().next(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId)); final long localCheckpoint = randomLongBetween(0, Long.MAX_VALUE - 1); @@ -1484,7 +1485,7 @@ public void testMissingActiveIdsDoesNotPreventAdvanceWithRemoteTranslogEnabled() assigned.putAll(active); assigned.putAll(initializing); AllocationId primaryId = active.keySet().iterator().next(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1514,7 +1515,7 @@ public void testMissingInSyncIdsDoesNotPreventAdvanceWithRemoteTranslogEnabled() logger.info("active: {}, initializing: {}", active, initializing); AllocationId primaryId = active.keySet().iterator().next(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1539,7 +1540,7 @@ public void testInSyncIdsAreIgnoredIfNotValidatedByClusterManagerWithRemoteTrans final Map initializing = randomAllocationsWithLocalCheckpoints(1, 5); final Map nonApproved = randomAllocationsWithLocalCheckpoints(1, 5); final AllocationId primaryId = active.keySet().iterator().next(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1577,7 +1578,7 @@ public void testInSyncIdsAreRemovedIfNotValidatedByClusterManagerWithRemoteTrans if (randomBoolean()) { allocations.putAll(initializingToBeRemoved); } - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1623,7 +1624,7 @@ public void testUpdateAllocationIdsFromClusterManagerWithRemoteTranslogEnabled() final Set initializingIds = activeAndInitializingAllocationIds.v2(); AllocationId primaryId = activeAllocationIds.iterator().next(); IndexShardRoutingTable routingTable = routingTable(initializingIds, primaryId); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1927,7 +1928,7 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() { } public void testPrimaryContextHandoffWithRemoteTranslogEnabled() throws IOException { - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings); final ShardId shardId = new ShardId("test", "_na_", 0); @@ -2106,7 +2107,7 @@ public void testPrimaryContextHandoffWithRemoteTranslogEnabled() throws IOExcept public void testIllegalStateExceptionIfUnknownAllocationIdWithRemoteTranslogEnabled() { final AllocationId active = AllocationId.newInitializing(); final AllocationId initializing = AllocationId.newInitializing(); - Settings settings = Settings.builder().put("index.remote_store.translog.enabled", "true").build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); final ReplicationTracker tracker = newTracker(active, settings); tracker.updateFromClusterManager( randomNonNegativeLong(), diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 5ddbf1971f114..915a15da6cb1d 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2697,8 +2697,7 @@ public void testRelocatedForRemoteTranslogBackedIndexWithAsyncDurability() throw .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC) .build(); @@ -2809,7 +2808,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test") .build(), new InternalEngineFactory() ); @@ -2848,6 +2848,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test1") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test1") .build(), new InternalEngineFactory() ); @@ -2888,6 +2890,8 @@ public void testRestoreShardFromRemoteStore(boolean performFlush) throws IOExcep Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") .build(), new InternalEngineFactory() ); @@ -4821,8 +4825,7 @@ public void testTranslogFactoryForRemoteTranslogBackedPrimaryShard() throws IOEx .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") .build(); final IndexShard primaryShard = newStartedShard(true, primarySettings, new NRTReplicationEngineFactory()); @@ -4837,8 +4840,7 @@ public void testTranslogFactoryForRemoteTranslogBackedReplicaShard() throws IOEx .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") .build(); final IndexShard replicaShard = newStartedShard(false, primarySettings, new NRTReplicationEngineFactory()); diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 6f38f080e5035..7c119bfbbc573 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -58,6 +58,8 @@ public void setup(boolean primary, int numberOfDocs) throws IOException { primary, Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(), new InternalEngineFactory() @@ -386,6 +388,8 @@ private Tuple m true, Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(), new InternalEngineFactory() diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index aef90483b4bbd..690c7955ff338 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -35,7 +35,6 @@ public class ReplicaRecoveryWithRemoteTranslogOnPrimaryTests extends OpenSearchI private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, "true") .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "100ms") .build(); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index aa590358efa25..94834825c2044 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -892,6 +892,8 @@ private void testNRTReplicaWithRemoteStorePromotedAsPrimary(boolean performFlush Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") .build(); try (ReplicationGroup shards = createGroup(1, settings, indexMapping, new NRTReplicationEngineFactory(), createTempDir())) { @@ -934,6 +936,8 @@ private void testNRTReplicaWithRemoteStorePromotedAsPrimary(boolean performFlush assertTrue(nextPrimary.translogStats().estimatedNumberOfOperations() >= additonalDocs); assertTrue(nextPrimary.translogStats().getUncommittedOperations() >= additonalDocs); + int prevOperationCount = nextPrimary.translogStats().estimatedNumberOfOperations(); + // promote the replica shards.promoteReplicaToPrimary(nextPrimary).get(); @@ -946,7 +950,7 @@ private void testNRTReplicaWithRemoteStorePromotedAsPrimary(boolean performFlush // As we are downloading segments from remote segment store on failover, there should not be // any operations replayed from translog - assertEquals(0, nextPrimary.translogStats().estimatedNumberOfOperations()); + assertEquals(prevOperationCount, nextPrimary.translogStats().estimatedNumberOfOperations()); // refresh and push segments to our other replica. nextPrimary.refresh("test"); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithRemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithRemoteIndexShardTests.java index cae15a2c53f3f..a67b60d6128d1 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithRemoteIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithRemoteIndexShardTests.java @@ -20,8 +20,8 @@ public class SegmentReplicationWithRemoteIndexShardTests extends OpenSearchIndex private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "temp-fs") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") .build(); public void testReplicaSyncingFromRemoteStore() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index ef23ce927eff9..682c9c53d10a0 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -61,7 +61,7 @@ public void setup() { public void testNewDirectory() throws IOException { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_UUID, "uuid_1") - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "remote_store_repository") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "remote_store_repository") .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); @@ -101,7 +101,9 @@ public void testNewDirectory() throws IOException { } public void testNewDirectoryRepositoryDoesNotExist() { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "remote_store_repository").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "remote_store_repository") + .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); ShardPath shardPath = new ShardPath(false, tempDir, tempDir, new ShardId(indexSettings.getIndex(), 0)); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index 74b1fab30d83f..d26379eaefa5c 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -182,7 +182,6 @@ private TranslogConfig getTranslogConfig(final Path path) { // only randomize between nog age retention and a long one, so failures will have a chance of reproducing .put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomBoolean() ? "-1ms" : "1h") .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomIntBetween(-1, 2048) + "b") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .build(); return getTranslogConfig(path, settings); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 96022315743c2..8135d9cd3718e 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -22,7 +22,6 @@ public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLe private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, "true") .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "100ms") .build(); diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java index deae4eaca19e7..04b5aa58ea485 100644 --- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java @@ -47,6 +47,8 @@ public class RemoteStoreReplicationSourceTests extends OpenSearchIndexLevelRepli private final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-repo") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index c750bf0f56e36..9d711d464754c 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -216,7 +216,8 @@ private Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) .build(); } diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index 21c3b30b24939..e3b1c7af6891c 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -532,7 +532,8 @@ protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) .build(); } From b669533ab9a13fc6075510d54311a065f3414567 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Fri, 21 Jul 2023 18:08:41 +0530 Subject: [PATCH 143/186] Mute test - SegmentReplicationIndexShardTests#testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit (#8818) Signed-off-by: bansvaru --- .../index/shard/SegmentReplicationIndexShardTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 94834825c2044..d988e34ef18dc 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -884,6 +884,7 @@ public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitRefresh() throws testNRTReplicaWithRemoteStorePromotedAsPrimary(true, false); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8817") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(true, true); } From 611ecc2ad50c7e1833e7fe8ef2a6cabeb6028c57 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Fri, 21 Jul 2023 18:58:51 +0530 Subject: [PATCH 144/186] [Remote Segment Store] Add Lucene major version to UploadedSegmentMetadata (#8088) * Add Lucene version to UploadedSegmentMetadata --------- Signed-off-by: Bhumika Saini --- .../index/remote/RemoteStoreUtils.java | 21 +++ .../store/RemoteSegmentStoreDirectory.java | 82 ++++++++++- .../index/remote/RemoteStoreUtilsTests.java | 22 +++ .../RemoteSegmentStoreDirectoryTests.java | 128 ++++++++++++++---- .../RemoteSegmentMetadataHandlerTests.java | 5 + 5 files changed, 225 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index 1eeadfe228a45..114d07589b0c0 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -48,4 +48,25 @@ public static long invertLong(String str) { } return Long.MAX_VALUE - num; } + + /** + * Extracts the segment name from the provided segment file name + * @param filename Segment file name to parse + * @return Name of the segment that the segment file belongs to + */ + public static String getSegmentName(String filename) { + // Segment file names follow patterns like "_0.cfe" or "_0_1_Lucene90_0.dvm". + // Here, the segment name is "_0", which is the set of characters + // starting with "_" until the next "_" or first ".". + int endIdx = filename.indexOf('_', 1); + if (endIdx == -1) { + endIdx = filename.indexOf('.'); + } + + if (endIdx == -1) { + throw new IllegalArgumentException("Unable to infer segment name for segment file " + filename); + } + + return filename.substring(0, endIdx); + } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index d3e8d961337cc..8ee267cb67e68 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -14,6 +14,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.store.ByteBuffersIndexOutput; @@ -22,6 +24,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Version; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.common.UUIDs; @@ -217,6 +220,14 @@ public static class UploadedSegmentMetadata { private final String checksum; private final long length; + /** + * The Lucene major version that wrote the original segment files. + * As part of the Lucene version compatibility check, this version information stored in the metadata + * will be used to skip downloading the segment files unnecessarily + * if they were written by an incompatible Lucene version. + */ + private int writtenByMajor; + UploadedSegmentMetadata(String originalFilename, String uploadedFilename, String checksum, long length) { this.originalFilename = originalFilename; this.uploadedFilename = uploadedFilename; @@ -226,7 +237,14 @@ public static class UploadedSegmentMetadata { @Override public String toString() { - return String.join(SEPARATOR, originalFilename, uploadedFilename, checksum, String.valueOf(length)); + return String.join( + SEPARATOR, + originalFilename, + uploadedFilename, + checksum, + String.valueOf(length), + String.valueOf(writtenByMajor) + ); } public String getChecksum() { @@ -239,12 +257,35 @@ public long getLength() { public static UploadedSegmentMetadata fromString(String uploadedFilename) { String[] values = uploadedFilename.split(SEPARATOR); - return new UploadedSegmentMetadata(values[0], values[1], values[2], Long.parseLong(values[3])); + UploadedSegmentMetadata metadata = new UploadedSegmentMetadata(values[0], values[1], values[2], Long.parseLong(values[3])); + if (values.length < 5) { + logger.error("Lucene version is missing for UploadedSegmentMetadata: " + uploadedFilename); + } + + metadata.setWrittenByMajor(Integer.parseInt(values[4])); + + return metadata; } public String getOriginalFilename() { return originalFilename; } + + public void setWrittenByMajor(int writtenByMajor) { + if (writtenByMajor <= Version.LATEST.major && writtenByMajor >= Version.MIN_SUPPORTED_MAJOR) { + this.writtenByMajor = writtenByMajor; + } else { + throw new IllegalArgumentException( + "Lucene major version supplied (" + + writtenByMajor + + ") is incorrect. Should be between Version.LATEST (" + + Version.LATEST.major + + ") and Version.MIN_SUPPORTED_MAJOR (" + + Version.MIN_SUPPORTED_MAJOR + + ")." + ); + } + } } /** @@ -582,10 +623,13 @@ public void uploadMetadata( ); try { try (IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT)) { + Map segmentToLuceneVersion = getSegmentToLuceneVersion(segmentFiles, segmentInfosSnapshot); Map uploadedSegments = new HashMap<>(); for (String file : segmentFiles) { if (segmentsUploadedToRemoteStore.containsKey(file)) { - uploadedSegments.put(file, segmentsUploadedToRemoteStore.get(file).toString()); + UploadedSegmentMetadata metadata = segmentsUploadedToRemoteStore.get(file); + metadata.setWrittenByMajor(segmentToLuceneVersion.get(metadata.originalFilename)); + uploadedSegments.put(file, metadata.toString()); } else { throw new NoSuchFileException(file); } @@ -615,6 +659,38 @@ public void uploadMetadata( } } + /** + * Parses the provided SegmentInfos to retrieve a mapping of the provided segment files to + * the respective Lucene major version that wrote the segments + * @param segmentFiles List of segment files for which the Lucene major version is needed + * @param segmentInfosSnapshot SegmentInfos instance to parse + * @return Map of the segment file to its Lucene major version + */ + private Map getSegmentToLuceneVersion(Collection segmentFiles, SegmentInfos segmentInfosSnapshot) { + Map segmentToLuceneVersion = new HashMap<>(); + for (SegmentCommitInfo segmentCommitInfo : segmentInfosSnapshot) { + SegmentInfo info = segmentCommitInfo.info; + Set segFiles = info.files(); + for (String file : segFiles) { + segmentToLuceneVersion.put(file, info.getVersion().major); + } + } + + for (String file : segmentFiles) { + if (segmentToLuceneVersion.containsKey(file) == false) { + if (file.equals(segmentInfosSnapshot.getSegmentsFileName())) { + segmentToLuceneVersion.put(file, segmentInfosSnapshot.getCommitLuceneVersion().major); + } else { + // Fallback to the Lucene major version of the respective segment's .si file + String segmentInfoFileName = RemoteStoreUtils.getSegmentName(file) + ".si"; + segmentToLuceneVersion.put(file, segmentToLuceneVersion.get(segmentInfoFileName)); + } + } + } + + return segmentToLuceneVersion; + } + /** * Try to delete file from local store. Fails silently on failures * @param filename: name of the file to be deleted diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index 5b9135afb66f3..9afa75dd601b2 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -38,4 +38,26 @@ public void testinvert() { assertEquals(num, RemoteStoreUtils.invertLong(RemoteStoreUtils.invertLong(num))); } } + + public void testGetSegmentNameForCfeFile() { + assertEquals("_foo", RemoteStoreUtils.getSegmentName("_foo.cfe")); + } + + public void testGetSegmentNameForDvmFile() { + assertEquals("_bar", RemoteStoreUtils.getSegmentName("_bar_1_Lucene90_0.dvm")); + } + + public void testGetSegmentNameWeirdSegmentNameOnlyUnderscore() { + // Validate behaviour when segment name contains delimiters only + assertEquals("_", RemoteStoreUtils.getSegmentName("_.dvm")); + } + + public void testGetSegmentNameUnderscoreDelimiterOverrides() { + // Validate behaviour when segment name contains delimiters only + assertEquals("_", RemoteStoreUtils.getSegmentName("___.dvm")); + } + + public void testGetSegmentNameException() { + assertThrows(IllegalArgumentException.class, () -> RemoteStoreUtils.getSegmentName("dvd")); + } } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 72d8c234d71c8..7c765cf5df0be 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -21,6 +21,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.OutputStreamIndexOutput; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.Version; import org.junit.After; import org.junit.Before; import org.mockito.Mockito; @@ -124,14 +125,42 @@ public void testUploadedSegmentMetadataToString() { "123456", 1234 ); - assertEquals("abc::pqr::123456::1234", metadata.toString()); + metadata.setWrittenByMajor(Version.LATEST.major); + assertEquals("abc::pqr::123456::1234::" + Version.LATEST.major, metadata.toString()); + } + + public void testUploadedSegmentMetadataToStringExceptionTooNew() { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = new RemoteSegmentStoreDirectory.UploadedSegmentMetadata( + "abc", + "pqr", + "123456", + 1234 + ); + assertThrows(IllegalArgumentException.class, () -> metadata.setWrittenByMajor(Version.LATEST.major + 1)); + } + + public void testUploadedSegmentMetadataToStringExceptionTooOld() { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = new RemoteSegmentStoreDirectory.UploadedSegmentMetadata( + "abc", + "pqr", + "123456", + 1234 + ); + assertThrows(IllegalArgumentException.class, () -> metadata.setWrittenByMajor(Version.LATEST.major - 2)); } public void testUploadedSegmentMetadataFromString() { RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString( - "_0.cfe::_0.cfe__uuidxyz::4567::372000" + "_0.cfe::_0.cfe__uuidxyz::4567::372000::" + Version.LATEST.major + ); + assertEquals("_0.cfe::_0.cfe__uuidxyz::4567::372000::" + Version.LATEST.major, metadata.toString()); + } + + public void testUploadedSegmentMetadataFromStringException() { + assertThrows( + ArrayIndexOutOfBoundsException.class, + () -> RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString("_0.cfe::_0.cfe__uuidxyz::4567::372000") ); - assertEquals("_0.cfe::_0.cfe__uuidxyz::4567::372000", metadata.toString()); } public void testGetPrimaryTermGenerationUuid() { @@ -176,6 +205,8 @@ private Map getDummyMetadata(String prefix, int commitGeneration + randomIntBetween(1000, 5000) + "::" + randomIntBetween(512000, 1024000) + + "::" + + Version.MIN_SUPPORTED_MAJOR ); metadata.put( prefix + ".cfs", @@ -188,6 +219,8 @@ private Map getDummyMetadata(String prefix, int commitGeneration + randomIntBetween(1000, 5000) + "::" + randomIntBetween(512000, 1024000) + + "::" + + Version.MIN_SUPPORTED_MAJOR ); metadata.put( prefix + ".si", @@ -200,6 +233,8 @@ private Map getDummyMetadata(String prefix, int commitGeneration + randomIntBetween(1000, 5000) + "::" + randomIntBetween(512000, 1024000) + + "::" + + Version.LATEST.major ); metadata.put( "segments_" + commitGeneration, @@ -213,6 +248,8 @@ private Map getDummyMetadata(String prefix, int commitGeneration + randomIntBetween(1000, 5000) + "::" + randomIntBetween(1024, 5120) + + "::" + + Version.LATEST.major ); return metadata; } @@ -611,8 +648,8 @@ public void testContainsFile() throws IOException { ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); - metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512"); - metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024"); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major); when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(createMetadataFileBytes(metadata, 1, 5)); @@ -641,7 +678,7 @@ public void testUploadMetadataEmpty() throws IOException { IndexOutput indexOutput = mock(IndexOutput.class); when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); - Collection segmentFiles = List.of("s1", "s2", "s3"); + Collection segmentFiles = List.of("_s1.si", "_s1.cfe", "_s3.cfs"); assertThrows( NoSuchFileException.class, () -> remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L, 34L) @@ -649,29 +686,45 @@ public void testUploadMetadataEmpty() throws IOException { } public void testUploadMetadataNonEmpty() throws IOException { - populateMetadata(); + indexDocs(142364, 5); + flushShard(indexShard, true); + SegmentInfos segInfos = indexShard.store().readLastCommittedSegmentsInfo(); + long primaryTerm = 12; + String primaryTermLong = RemoteStoreUtils.invertLong(primaryTerm); + long generation = segInfos.getGeneration(); + String generationLong = RemoteStoreUtils.invertLong(generation); + String latestMetadataFileName = "metadata__" + primaryTermLong + "__" + generationLong + "__abc"; + List metadataFiles = List.of(latestMetadataFileName); + when( + remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, + 1 + ) + ).thenReturn(metadataFiles); + Map> metadataFilenameContentMapping = Map.of( + latestMetadataFileName, + getDummyMetadata("_0", (int) generation) + ); + when(remoteMetadataDirectory.openInput(latestMetadataFileName, IOContext.DEFAULT)).thenReturn( + createMetadataFileBytes(metadataFilenameContentMapping.get(latestMetadataFileName), generation, primaryTerm) + ); + remoteSegmentStoreDirectory.init(); Directory storeDirectory = mock(Directory.class); BytesStreamOutput output = new BytesStreamOutput(); IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + when(storeDirectory.createOutput(startsWith("metadata__" + primaryTermLong + "__" + generationLong), eq(IOContext.DEFAULT))) + .thenReturn(indexOutput); - String generation = RemoteStoreUtils.invertLong(segmentInfos.getGeneration()); - String primaryTerm = RemoteStoreUtils.invertLong(12); - when(storeDirectory.createOutput(startsWith("metadata__" + primaryTerm + "__" + generation), eq(IOContext.DEFAULT))).thenReturn( - indexOutput - ); - - Collection segmentFiles = List.of("_0.si", "_0.cfe", "_0.cfs", "segments_1"); - remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L, 34L); + remoteSegmentStoreDirectory.uploadMetadata(segInfos.files(true), segInfos, storeDirectory, primaryTerm, generation); verify(remoteMetadataDirectory).copyFrom( eq(storeDirectory), - startsWith("metadata__" + primaryTerm + "__" + generation), - startsWith("metadata__" + primaryTerm + "__" + generation), + startsWith("metadata__" + primaryTermLong + "__" + generationLong), + startsWith("metadata__" + primaryTermLong + "__" + generationLong), eq(IOContext.DEFAULT) ); - VersionedCodecStreamWrapper streamWrapper = new VersionedCodecStreamWrapper<>( new RemoteSegmentMetadataHandler(), RemoteSegmentMetadata.CURRENT_VERSION, @@ -680,16 +733,25 @@ public void testUploadMetadataNonEmpty() throws IOException { RemoteSegmentMetadata remoteSegmentMetadata = streamWrapper.readStream( new ByteArrayIndexInput("expected", BytesReference.toBytes(output.bytes())) ); - Map actual = remoteSegmentStoreDirectory .getSegmentsUploadedToRemoteStore(); Map expected = remoteSegmentMetadata.getMetadata(); - for (String filename : expected.keySet()) { assertEquals(expected.get(filename).toString(), actual.get(filename).toString()); } } + public void testUploadMetadataNoSegmentCommitInfos() throws IOException { + SegmentInfos segInfos = indexShard.store().readLastCommittedSegmentsInfo(); + int numSegCommitInfos = segInfos.size(); + assertEquals( + "For a fresh index, the number of SegmentCommitInfo instances associated with the SegmentInfos instance should be 0, but were found to be " + + numSegCommitInfos, + 0, + numSegCommitInfos + ); + } + public void testNoMetadataHeaderCorruptIndexException() throws IOException { List metadataFiles = List.of(metadataFilename); when( @@ -700,8 +762,8 @@ public void testNoMetadataHeaderCorruptIndexException() throws IOException { ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); - metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); - metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major); BytesStreamOutput output = new BytesStreamOutput(); OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); @@ -723,8 +785,8 @@ public void testInvalidCodecHeaderCorruptIndexException() throws IOException { ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); - metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); - metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major); BytesStreamOutput output = new BytesStreamOutput(); OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); @@ -748,8 +810,8 @@ public void testHeaderMinVersionCorruptIndexException() throws IOException { ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); - metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); - metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major); BytesStreamOutput output = new BytesStreamOutput(); OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); @@ -773,8 +835,8 @@ public void testHeaderMaxVersionCorruptIndexException() throws IOException { ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); - metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); - metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major); BytesStreamOutput output = new BytesStreamOutput(); OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); @@ -798,8 +860,8 @@ public void testIncorrectChecksumCorruptIndexException() throws IOException { ).thenReturn(metadataFiles); Map metadata = new HashMap<>(); - metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512"); - metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024"); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major); BytesStreamOutput output = new BytesStreamOutput(); IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); @@ -947,6 +1009,12 @@ public void testSegmentMetadataCurrentVersion() { assertEquals(RemoteSegmentMetadata.CURRENT_VERSION, 1); } + private void indexDocs(int startDocId, int numberOfDocs) throws IOException { + for (int i = startDocId; i < startDocId + numberOfDocs; i++) { + indexDoc(indexShard, "_doc", Integer.toString(i)); + } + } + public void testMetadataFileNameOrder() { String file1 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 21, 23, 1, 1); String file2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 38, 1, 1); diff --git a/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java b/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java index dc7940529f024..2fee77ab563c0 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.store.ByteBuffersIndexOutput; import org.apache.lucene.store.OutputStreamIndexOutput; +import org.apache.lucene.util.Version; import org.junit.After; import org.junit.Before; import org.opensearch.cluster.metadata.IndexMetadata; @@ -134,6 +135,8 @@ private Map getDummyData() { + randomIntBetween(1000, 5000) + "::" + randomIntBetween(1024, 2048) + + "::" + + Version.LATEST.major ); expectedOutput.put( prefix + ".cfs", @@ -146,6 +149,8 @@ private Map getDummyData() { + randomIntBetween(1000, 5000) + "::" + randomIntBetween(1024, 2048) + + "::" + + Version.LATEST.major ); return expectedOutput; } From 3adde3a1182a78da6ca376a5cbc679c83e783d32 Mon Sep 17 00:00:00 2001 From: Rohit Ashiwal Date: Fri, 21 Jul 2023 21:23:40 +0530 Subject: [PATCH 145/186] fixtures: fix read-only krb5kdc composeBuild failure (#8799) Running composeBuild task for krb5kdc-fixture throws read-only file system error while writing hostname to local hosts file. Since, hostname resolution is already present and pointing to local node, let's remove writing it again from the Dockerfile. See: #8761 Signed-off-by: Rohit Ashiwal --- test/fixtures/krb5kdc-fixture/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/krb5kdc-fixture/Dockerfile b/test/fixtures/krb5kdc-fixture/Dockerfile index 55ffacbdedb8a..c194f2637a2d9 100644 --- a/test/fixtures/krb5kdc-fixture/Dockerfile +++ b/test/fixtures/krb5kdc-fixture/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:14.04 RUN apt update -y RUN apt upgrade -y ADD . /fixture -RUN echo kerberos.build.opensearch.org > /etc/hostname && echo "127.0.0.1 kerberos.build.opensearch.org" >> /etc/hosts +RUN echo kerberos.build.opensearch.org > /etc/hostname RUN bash /fixture/src/main/resources/provision/installkdc.sh EXPOSE 88 From 78537a286dea58c61a243cbfcbcf1717529a2ade Mon Sep 17 00:00:00 2001 From: Ketan Verma <9292653+ketanv3@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:47:02 -0400 Subject: [PATCH 146/186] Exclude 'benchmarks' from codecov report (#8805) Signed-off-by: Ketan Verma --- CHANGELOG.md | 1 + codecov.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f809f1f83c88..0ab07011aefee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) - Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) - Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) +- Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) ### Deprecated diff --git a/codecov.yml b/codecov.yml index b8a7d220fa214..dac8f30956846 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,6 +3,7 @@ codecov: ignore: - "test" + - "benchmarks" coverage: precision: 2 From db838c4ca090103a8ba562b3cb124044be725c62 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Fri, 21 Jul 2023 16:15:19 -0500 Subject: [PATCH 147/186] Remove searchSegmentOrderReversed from IndexSettings (#8808) This property is unused now and can be safely removed. Signed-off-by: Andrew Ross --- .../java/org/opensearch/index/IndexSettings.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 1fb036580af27..0749ad0876534 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -680,7 +680,6 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingTotalFieldsLimit; private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; - private volatile boolean searchSegmentOrderReversed; /** * The maximum number of refresh listeners allows on this shard. @@ -921,10 +920,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti ); } - private void setSearchSegmentOrderReversed(boolean reversed) { - this.searchSegmentOrderReversed = reversed; - } - private void setSearchIdleAfter(TimeValue searchIdleAfter) { if (this.replicationType == ReplicationType.SEGMENT && this.getNumberOfReplicas() > 0) { logger.warn("Search idle is not supported for indices with replicas using 'replication.type: SEGMENT'"); @@ -1109,13 +1104,6 @@ public Settings getNodeSettings() { return nodeSettings; } - /** - * Returns true if index level setting for leaf reverse order search optimization is enabled - */ - public boolean getSearchSegmentOrderReversed() { - return this.searchSegmentOrderReversed; - } - /** * Updates the settings and index metadata and notifies all registered settings consumers with the new settings iff at least one * setting has changed. From 77696825ed5388eb9135dd4b2d3a6c2c801f8092 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sun, 23 Jul 2023 11:57:23 +0530 Subject: [PATCH 148/186] Adding @sohami to OpenSearch maintainers (#8828) I have nominated and maintainers have agreed to invite Sorabh Hamirwasia(@sohami) to be a co-maintainer. Sorabh has kindly accepted. Sorabh has led the design and implementation of multiple features like query cancellation, concurrent segment search for aggregations and snapshot inter-operability w/remote storage in OpenSearch. Some significant issues and PRs authored by Sorabh for this effort are as follows: Feature proposals Concurrent segment search for aggregations : #6798 Searchable Remote Index : #2900 Implementations Concurrent segment search for aggregations: #7514 Lucene changes to leaf slices for concurrent search: apache/lucene#12374 Moving concurrent search to core : #7203 Query cancellation support : #986 In total, Sorabh has authored 14 PRs going back to Aug 2021. He also frequently reviews contributions from others and has reviewed nearly 20 PRs in the same time frame. I believe Sorabh will be a valuable addition as a maintainer of OpenSearch and will continue to contribute to the success of the project going forward. Signed-off-by: Bukhtawar Khan --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e5d34bfc1e6dd..1a108c35429ae 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale +* @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale @sohami diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 8027c5daffc69..95e87f4be43bf 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -25,6 +25,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Sachin Kale | [sachinpkale](https://github.com/sachinpkale) | Amazon | | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon | | Shweta Thareja | [shwetathareja](https://github.com/shwetathareja) | Amazon | +| Sorabh Hamirwasia | [sohami](https://github.com/sohami) | Amazon | | Suraj Singh | [dreamer-89](https://github.com/dreamer-89) | Amazon | | Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon | | Vacha Shah | [VachaShah](https://github.com/VachaShah) | Amazon | From a29521eaf258862bd68f1c40a5cd28a03d109a64 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Mon, 24 Jul 2023 15:07:47 +0530 Subject: [PATCH 149/186] Fix flaky test - org.opensearch.repositories.blobstore.BlobStoreRepositoryTests (#8835) Signed-off-by: bansvaru --- .../repositories/blobstore/BlobStoreRepositoryTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 9d711d464754c..28513f279f8ad 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -32,6 +32,7 @@ package org.opensearch.repositories.blobstore; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.Version; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -90,6 +91,7 @@ /** * Tests for the {@link BlobStoreRepository} and its subclasses. */ +@LuceneTestCase.SuppressFileSystems("ExtrasFS") public class BlobStoreRepositoryTests extends OpenSearchSingleNodeTestCase { static final String REPO_TYPE = "fsLike"; From d9abf12f1aaa505c530adcd1d6425a75d2de4289 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:07:59 -0700 Subject: [PATCH 150/186] Bump com.netflix.nebula.ospackage-base from 11.3.0 to 11.4.0 in /distribution/packages (#8838) * Bump com.netflix.nebula.ospackage-base in /distribution/packages Bumps com.netflix.nebula.ospackage-base from 11.3.0 to 11.4.0. --- updated-dependencies: - dependency-name: com.netflix.nebula.ospackage-base dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 3 ++- distribution/packages/build.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ab07011aefee..7d4325cd3c2cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) - Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) +- Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) @@ -98,4 +99,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index a0ae4c5c2472a..7914fcc172ef4 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -63,7 +63,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.3.0" + id "com.netflix.nebula.ospackage-base" version "11.4.0" } void addProcessFilesTask(String type, boolean jdk) { From ea68f82e4f6f0ac70986866864927fae41b541d9 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 24 Jul 2023 13:23:16 -0400 Subject: [PATCH 151/186] Manually Bump org.gradle.test-retry from 1.5.3 to 1.5.4 (#8842) * Bump retry Signed-off-by: Stephen Crawford * Add pr num Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> --- CHANGELOG.md | 1 + build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d4325cd3c2cf..d5b3fe41d830f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) - Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) +- Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) ### Changed diff --git a/build.gradle b/build.gradle index 6a14ab231894b..88c485dae9350 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ plugins { id 'opensearch.docker-support' id 'opensearch.global-build-info' id "com.diffplug.spotless" version "6.19.0" apply false - id "org.gradle.test-retry" version "1.5.3" apply false + id "org.gradle.test-retry" version "1.5.4" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' } From 8058f9e1640c57cdf04555d2e167f74bc16f8774 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:07:15 -0700 Subject: [PATCH 152/186] Bump com.google.http-client:google-http-client-gson from 1.43.2 to 1.43.3 in /plugins/repository-gcs (#8840) * Bump com.google.http-client:google-http-client-gson Bumps [com.google.http-client:google-http-client-gson](https://github.com/googleapis/google-http-java-client) from 1.43.2 to 1.43.3. - [Release notes](https://github.com/googleapis/google-http-java-client/releases) - [Changelog](https://github.com/googleapis/google-http-java-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-http-java-client/compare/v1.43.2...v1.43.3) --- updated-dependencies: - dependency-name: com.google.http-client:google-http-client-gson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- .../licenses/google-http-client-gson-1.43.2.jar.sha1 | 1 - .../licenses/google-http-client-gson-1.43.3.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d5b3fe41d830f..a6d4b258c7427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) - Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) +- Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 1e8e358833cc9..fd2f713dd33cd 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -77,7 +77,7 @@ dependencies { api 'com.google.http-client:google-http-client:1.43.2' api 'com.google.http-client:google-http-client-appengine:1.43.2' - api 'com.google.http-client:google-http-client-gson:1.43.2' + api 'com.google.http-client:google-http-client-gson:1.43.3' api 'com.google.http-client:google-http-client-jackson2:1.43.2' api 'com.google.oauth-client:google-oauth-client:1.34.1' diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 deleted file mode 100644 index df0374aa27c70..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b1c2e3e89804c113dba7b342aa8e0fc2cf3d9378 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 new file mode 100644 index 0000000000000..43f4fe4a127e1 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 @@ -0,0 +1 @@ +252e267acf720ef6333488740a696a1d5e204639 \ No newline at end of file From 3ee42920fa886bdf566f1bb7b1666ffa72d36f54 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:48:16 -0400 Subject: [PATCH 153/186] Manually bump org.apache.zookeeper:zookeeper from 3.8.1 to 3.8.2 (#8844) * Update Zoo Signed-off-by: Stephen Crawford * Add pr num Signed-off-by: Stephen Crawford --------- Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6d4b258c7427..0ccffb74efc57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) - Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) +- Bump `org.apache.zookeeper:zookeeper` from 3.8.1 to 3.8.2 ([#8844](https://github.com/opensearch-project/OpenSearch/pull/8844)) - Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) - Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b6dc8820a85c5..b052988f88a2b 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -61,7 +61,7 @@ dependencies { api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" api "org.eclipse.jetty:jetty-server:${versions.jetty}" api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}" - api 'org.apache.zookeeper:zookeeper:3.8.1' + api 'org.apache.zookeeper:zookeeper:3.8.2' api "org.apache.commons:commons-text:1.10.0" api "commons-net:commons-net:3.9.0" runtimeOnly "com.google.guava:guava:${versions.guava}" From 3f8e0cd3de219dee302788a3616ec33835055e5f Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Mon, 24 Jul 2023 15:30:56 -0500 Subject: [PATCH 154/186] [Refactor] MediaTypeParser to MediaTypeParserRegistry (#8636) Refactors the static MediaTypeParser utility to a dynamic registry to support registration of new MediaType definitions. This is a next step to decoupling the x-content library from core classes in the server module to support modularity and cloud or serverless implementations. Signed-off-by: Nicholas Walter Knize --- CHANGELOG.md | 1 + .../noop/action/bulk/RestNoopBulkAction.java | 2 +- .../opensearch/client/RequestConverters.java | 63 ++++------- .../client/RestHighLevelClient.java | 8 +- .../client/indices/CreateIndexRequest.java | 104 ++---------------- .../client/indices/PutMappingRequest.java | 44 +------- .../org/opensearch/client/IngestClientIT.java | 5 +- .../client/RequestConvertersTests.java | 3 +- .../indices/CreateIndexRequestTests.java | 4 +- .../core/common/io/stream/StreamInput.java | 6 + .../opensearch/core/xcontent/MediaType.java | 43 +++++++- ...rser.java => MediaTypeParserRegistry.java} | 88 +++++++++------ .../common/xcontent/XContentFactory.java | 4 +- .../common/xcontent/XContentType.java | 65 ++--------- .../common/xcontent/MediaTypeParserTests.java | 30 ++--- .../action/PainlessExecuteAction.java | 34 +++--- .../opensearch/index/reindex/Reindexer.java | 19 ++-- .../reindex/TransportUpdateByQueryAction.java | 2 +- .../reindex/remote/RemoteResponseParsers.java | 14 +-- .../remote/RemoteScrollableHitSource.java | 14 +-- .../index/reindex/RestReindexActionTests.java | 2 +- .../opensearch/search/geo/GeoFilterIT.java | 4 +- .../storedscripts/PutStoredScriptRequest.java | 28 +++-- .../opensearch/action/bulk/BulkRequest.java | 26 ++--- .../action/bulk/BulkRequestBuilder.java | 6 +- .../action/bulk/BulkRequestParser.java | 17 +-- .../action/bulk/TransportShardBulkAction.java | 4 +- .../opensearch/action/index/IndexRequest.java | 53 +++++---- .../action/ingest/PutPipelineRequest.java | 40 +++---- .../ingest/SimulatePipelineRequest.java | 28 +++-- .../search/PutSearchPipelineRequest.java | 23 ++-- .../termvectors/TermVectorsRequest.java | 26 +++-- .../action/update/TransportUpdateAction.java | 4 +- .../action/update/UpdateHelper.java | 5 +- .../action/update/UpdateRequest.java | 9 +- .../common/xcontent/XContentHelper.java | 1 + .../extensions/rest/ExtensionRestRequest.java | 54 +++++---- .../rest/RestInitializeExtensionAction.java | 2 +- .../rest/RestSendToExtensionAction.java | 4 +- .../org/opensearch/index/IndexingSlowLog.java | 4 +- .../index/mapper/DocumentParser.java | 8 +- .../index/mapper/ParsedDocument.java | 13 ++- .../index/mapper/SourceFieldMapper.java | 10 +- .../index/mapper/SourceToParse.java | 16 +-- .../index/query/MoreLikeThisQueryBuilder.java | 29 +++-- .../reindex/ClientScrollableHitSource.java | 4 +- .../index/reindex/ScrollableHitSource.java | 13 ++- .../opensearch/index/shard/IndexShard.java | 3 +- .../index/termvectors/TermVectorsService.java | 6 +- .../org/opensearch/ingest/IngestService.java | 4 +- .../ingest/PipelineConfiguration.java | 35 +++--- .../opensearch/rest/AbstractRestChannel.java | 28 +++-- .../java/org/opensearch/rest/RestChannel.java | 6 +- .../org/opensearch/rest/RestController.java | 18 +-- .../java/org/opensearch/rest/RestRequest.java | 37 ++++--- .../cluster/RestPutStoredScriptAction.java | 8 +- .../admin/indices/RestCreateIndexAction.java | 2 +- .../indices/RestPutIndexTemplateAction.java | 2 +- .../admin/indices/RestPutMappingAction.java | 2 +- .../opensearch/rest/action/cat/RestTable.java | 10 +- .../rest/action/document/RestBulkAction.java | 2 +- .../action/document/RestGetSourceAction.java | 2 +- .../rest/action/document/RestIndexAction.java | 2 +- .../action/ingest/RestPutPipelineAction.java | 4 +- .../ingest/RestSimulatePipelineAction.java | 4 +- .../action/search/RestMultiSearchAction.java | 4 +- .../search/RestPutSearchPipelineAction.java | 4 +- .../opensearch/script/StoredScriptSource.java | 72 ++++++------ .../opensearch/search/fetch/FetchPhase.java | 3 +- .../search/lookup/SourceLookup.java | 6 +- .../pipeline/PipelineConfiguration.java | 35 +++--- .../pipeline/SearchPipelineService.java | 4 +- .../transport/TransportService.java | 5 + .../reroute/ClusterRerouteRequestTests.java | 2 +- .../PutStoredScriptRequestTests.java | 4 +- .../ingest/PutPipelineRequestTests.java | 4 +- .../common/xcontent/XContentTypeTests.java | 63 ++++++----- .../index/mapper/DynamicMappingTests.java | 4 +- .../ingest/PipelineConfigurationTests.java | 8 +- .../org/opensearch/rest/RestRequestTests.java | 8 +- .../script/ScriptMetadataTests.java | 11 +- .../org/opensearch/test/RandomObjects.java | 8 +- .../opensearch/test/rest/FakeRestRequest.java | 8 +- .../test/rest/OpenSearchRestTestCase.java | 11 +- .../yaml/ClientYamlTestExecutionContext.java | 29 ++--- .../rest/yaml/ClientYamlTestResponse.java | 5 +- .../opensearch/test/rest/yaml/ObjectPath.java | 6 +- 87 files changed, 712 insertions(+), 753 deletions(-) rename libs/core/src/main/java/org/opensearch/core/xcontent/{MediaTypeParser.java => MediaTypeParserRegistry.java} (53%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ccffb74efc57..3552c66c83229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) - Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) - Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) +- [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636)) ### Deprecated diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java index 6a75837f8a0fd..332e089ad9e76 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java @@ -99,7 +99,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC defaultPipeline, defaultRequireAlias, true, - request.getXContentType() + request.getMediaType() ); // short circuit the call to the transport layer diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java index f3fbadf07d304..d23a5976fada6 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java @@ -157,7 +157,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { // Bulk API only supports newline delimited JSON or Smile. Before executing // the bulk, we need to check that all requests have the same content-type // and this content-type is supported by the Bulk API. - XContentType bulkContentType = null; + MediaType bulkContentType = null; for (int i = 0; i < bulkRequest.numberOfActions(); i++) { DocWriteRequest action = bulkRequest.requests().get(i); @@ -245,7 +245,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { IndexRequest indexRequest = (IndexRequest) action; BytesReference indexSource = indexRequest.source(); - XContentType indexXContentType = indexRequest.getContentType(); + MediaType mediaType = indexRequest.getContentType(); try ( XContentParser parser = XContentHelper.createParser( @@ -257,7 +257,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, - indexXContentType + mediaType ) ) { try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) { @@ -266,7 +266,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { } } } else if (opType == DocWriteRequest.OpType.UPDATE) { - source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef(); + source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, ToXContent.EMPTY_PARAMS, false).toBytesRef(); } if (source != null) { @@ -392,30 +392,30 @@ static Request update(UpdateRequest updateRequest) throws IOException { // set for the partial document and the upsert document. This client // only accepts update requests that have the same content types set // for both doc and upsert. - XContentType xContentType = null; + MediaType mediaType = null; if (updateRequest.doc() != null) { - xContentType = updateRequest.doc().getContentType(); + mediaType = updateRequest.doc().getContentType(); } if (updateRequest.upsertRequest() != null) { - XContentType upsertContentType = updateRequest.upsertRequest().getContentType(); - if ((xContentType != null) && (xContentType != upsertContentType)) { + MediaType upsertContentType = updateRequest.upsertRequest().getContentType(); + if ((mediaType != null) && (mediaType != upsertContentType)) { throw new IllegalStateException( "Update request cannot have different content types for doc [" - + xContentType + + mediaType + "]" + " and upsert [" + upsertContentType + "] documents" ); } else { - xContentType = upsertContentType; + mediaType = upsertContentType; } } - if (xContentType == null) { - xContentType = Requests.INDEX_CONTENT_TYPE; + if (mediaType == null) { + mediaType = Requests.INDEX_CONTENT_TYPE; } request.addParameters(parameters.asMap()); - request.setEntity(createEntity(updateRequest, xContentType)); + request.setEntity(createEntity(updateRequest, mediaType)); return request; } @@ -816,14 +816,13 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest) return request; } - static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { - return createEntity(toXContent, xContentType, ToXContent.EMPTY_PARAMS); + static HttpEntity createEntity(ToXContent toXContent, MediaType mediaType) throws IOException { + return createEntity(toXContent, mediaType, ToXContent.EMPTY_PARAMS); } - static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType, ToXContent.Params toXContentParams) - throws IOException { - BytesRef source = XContentHelper.toXContent(toXContent, xContentType, toXContentParams, false).toBytesRef(); - return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); + static HttpEntity createEntity(ToXContent toXContent, MediaType mediaType, ToXContent.Params toXContentParams) throws IOException { + BytesRef source = XContentHelper.toXContent(toXContent, mediaType, toXContentParams, false).toBytesRef(); + return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(mediaType)); } static String endpoint(String index, String id) { @@ -868,20 +867,6 @@ static String endpoint(String[] indices, String endpoint, String type) { return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build(); } - /** - * Returns a {@link ContentType} from a given {@link XContentType}. - * - * @param xContentType the {@link XContentType} - * @return the {@link ContentType} - * - * @deprecated use {@link #createContentType(MediaType)} instead - */ - @Deprecated - @SuppressForbidden(reason = "Only allowed place to convert a XContentType to a ContentType") - public static ContentType createContentType(final XContentType xContentType) { - return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null); - } - /** * Returns a {@link ContentType} from a given {@link XContentType}. * @@ -1265,8 +1250,8 @@ Params withWaitForEvents(Priority waitForEvents) { * * @return the {@link IndexRequest}'s content type */ - static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) { - XContentType requestContentType = indexRequest.getContentType(); + static MediaType enforceSameContentType(IndexRequest indexRequest, @Nullable MediaType mediaType) { + MediaType requestContentType = indexRequest.getContentType(); if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE) { throw new IllegalArgumentException( "Unsupported content-type found for request with content-type [" @@ -1274,19 +1259,19 @@ static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable + "], only JSON and SMILE are supported" ); } - if (xContentType == null) { + if (mediaType == null) { return requestContentType; } - if (requestContentType != xContentType) { + if (requestContentType != mediaType) { throw new IllegalArgumentException( "Mismatching content-type found for request with content-type [" + requestContentType + "], previous requests have content-type [" - + xContentType + + mediaType + "]" ); } - return xContentType; + return mediaType; } /** diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index 9574d4b17819d..1ff5c81c4a0e3 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -88,9 +88,9 @@ import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.rankeval.RankEvalRequest; import org.opensearch.index.rankeval.RankEvalResponse; import org.opensearch.index.reindex.BulkByScrollResponse; @@ -2227,11 +2227,11 @@ protected final Resp parseEntity(final HttpEntity entity, final CheckedFu if (entity.getContentType() == null) { throw new IllegalStateException("OpenSearch didn't return the [Content-Type] header, unable to parse response body"); } - XContentType xContentType = XContentType.fromMediaType(entity.getContentType()); - if (xContentType == null) { + MediaType medaiType = MediaType.fromMediaType(entity.getContentType()); + if (medaiType == null) { throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType()); } - try (XContentParser parser = xContentType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) { + try (XContentParser parser = medaiType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) { return entityParser.apply(parser); } } diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java index 3ed41684b090e..3405e7e81e122 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java @@ -44,10 +44,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -77,7 +77,7 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX private Settings settings = EMPTY_SETTINGS; private BytesReference mappings; - private XContentType mappingsXContentType; + private MediaType mappingsMediaType; private final Set aliases = new HashSet<>(); @@ -123,17 +123,6 @@ public CreateIndexRequest settings(Settings settings) { return this; } - /** - * The settings to create the index with (either json or yaml format) - * - * @deprecated use {@link #settings(String source, MediaType mediaType)} instead - */ - @Deprecated - public CreateIndexRequest settings(String source, XContentType xContentType) { - this.settings = Settings.builder().loadFromSource(source, xContentType).build(); - return this; - } - /** * The settings to create the index with (either json or yaml format) */ @@ -162,23 +151,8 @@ public BytesReference mappings() { return mappings; } - public XContentType mappingsXContentType() { - return mappingsXContentType; - } - - /** - * Adds mapping that will be added when the index gets created. - * - * Note that the definition should *not* be nested under a type name. - * - * @param source The mapping source - * @param xContentType The content type of the source - * - * @deprecated use {@link #mapping(String source, MediaType mediaType)} instead - */ - @Deprecated - public CreateIndexRequest mapping(String source, XContentType xContentType) { - return mapping(new BytesArray(source), xContentType); + public MediaType mappingsMediaType() { + return mappingsMediaType; } /** @@ -213,7 +187,7 @@ public CreateIndexRequest mapping(XContentBuilder source) { */ public CreateIndexRequest mapping(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeParserRegistry.getDefaultMediaType()); builder.map(source); return mapping(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { @@ -221,24 +195,6 @@ public CreateIndexRequest mapping(Map source) { } } - /** - * Adds mapping that will be added when the index gets created. - * - * Note that the definition should *not* be nested under a type name. - * - * @param source The mapping source - * @param xContentType the content type of the mapping source - * - * @deprecated use {@link #mapping(BytesReference source, MediaType mediaType)} instead - */ - @Deprecated - public CreateIndexRequest mapping(BytesReference source, XContentType xContentType) { - Objects.requireNonNull(xContentType); - mappings = source; - mappingsXContentType = xContentType; - return this; - } - /** * Adds mapping that will be added when the index gets created. * @@ -250,7 +206,7 @@ public CreateIndexRequest mapping(BytesReference source, XContentType xContentTy public CreateIndexRequest mapping(BytesReference source, MediaType mediaType) { Objects.requireNonNull(mediaType); mappings = source; - mappingsXContentType = XContentType.fromMediaType(mediaType); + mappingsMediaType = mediaType; return this; } @@ -278,16 +234,6 @@ public CreateIndexRequest aliases(XContentBuilder source) { return aliases(BytesReference.bytes(source), source.contentType()); } - /** - * Sets the aliases that will be associated with the index when it gets created - * - * @deprecated use {@link #aliases(String, MediaType)} instead - */ - @Deprecated - public CreateIndexRequest aliases(String source, XContentType contentType) { - return aliases(new BytesArray(source), contentType); - } - /** * Sets the aliases that will be associated with the index when it gets created */ @@ -295,16 +241,6 @@ public CreateIndexRequest aliases(String source, MediaType mediaType) { return aliases(new BytesArray(source), mediaType); } - /** - * Sets the aliases that will be associated with the index when it gets created - * - * @deprecated use {@link #aliases(BytesReference source, MediaType contentType)} instead - */ - @Deprecated - public CreateIndexRequest aliases(BytesReference source, XContentType contentType) { - return aliases(source, (MediaType) contentType); - } - /** * Sets the aliases that will be associated with the index when it gets created */ @@ -345,18 +281,6 @@ public CreateIndexRequest aliases(Collection aliases) { return this; } - /** - * Sets the settings and mappings as a single source. - * - * Note that the mapping definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(String, MediaType)} instead - */ - @Deprecated - public CreateIndexRequest source(String source, XContentType xContentType) { - return source(new BytesArray(source), xContentType); - } - /** * Sets the settings and mappings as a single source. * @@ -375,20 +299,6 @@ public CreateIndexRequest source(XContentBuilder source) { return source(BytesReference.bytes(source), source.contentType()); } - /** - * Sets the settings and mappings as a single source. - * - * Note that the mapping definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(BytesReference, MediaType)} instead - */ - @Deprecated - public CreateIndexRequest source(BytesReference source, XContentType xContentType) { - Objects.requireNonNull(xContentType); - source(XContentHelper.convertToMap(source, false, xContentType).v2()); - return this; - } - /** * Sets the settings and mappings as a single source. * @@ -458,7 +368,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (mappings != null) { try (InputStream stream = mappings.streamInput()) { - builder.rawField(MAPPINGS.getPreferredName(), stream, mappingsXContentType); + builder.rawField(MAPPINGS.getPreferredName(), stream, mappingsMediaType); } } diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java index a2237efaed25f..d17dc54713789 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java @@ -39,8 +39,8 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -98,17 +98,7 @@ public BytesReference source() { } /** - * The {@link XContentType} of the mapping source. - * - * @deprecated use {@link #mediaType()} instead - */ - @Deprecated - public XContentType xContentType() { - return XContentType.fromMediaType(mediaType); - } - - /** - * The {@link XContentType} of the mapping source. + * The {@link MediaType} of the mapping source. */ public MediaType mediaType() { return mediaType; @@ -121,7 +111,7 @@ public MediaType mediaType() { */ public PutMappingRequest source(Map mappingSource) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeParserRegistry.getDefaultMediaType()); builder.map(mappingSource); return source(builder); } catch (IOException e) { @@ -129,20 +119,6 @@ public PutMappingRequest source(Map mappingSource) { } } - /** - * The mapping source definition. - * - * Note that the definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(String, MediaType)} instead - */ - @Deprecated - public PutMappingRequest source(String mappingSource, XContentType xContentType) { - this.source = new BytesArray(mappingSource); - this.mediaType = xContentType; - return this; - } - /** * The mapping source definition. * @@ -165,20 +141,6 @@ public PutMappingRequest source(XContentBuilder builder) { return this; } - /** - * The mapping source definition. - * - * Note that the definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(BytesReference, MediaType)} instead - */ - @Deprecated - public PutMappingRequest source(BytesReference source, XContentType xContentType) { - this.source = source; - this.mediaType = xContentType; - return this; - } - /** * The mapping source definition. * diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java index 84cd2fb8b20dd..e3c8197dc2c90 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java @@ -161,10 +161,7 @@ private void testSimulatePipeline(boolean isVerbose, boolean isFailure) throws I } builder.endObject(); - SimulatePipelineRequest request = new SimulatePipelineRequest( - BytesReference.bytes(builder), - XContentType.fromMediaType(builder.contentType()) - ); + SimulatePipelineRequest request = new SimulatePipelineRequest(BytesReference.bytes(builder), builder.contentType()); request.setVerbose(isVerbose); SimulatePipelineResponse response = execute( request, diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java index ee6aeeeacf633..15a99b3e91685 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java @@ -76,6 +76,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -809,7 +810,7 @@ public void testUpdate() throws IOException { UpdateRequest parsedUpdateRequest = new UpdateRequest(); - XContentType entityContentType = XContentType.fromMediaType(entity.getContentType()); + MediaType entityContentType = MediaType.fromMediaType(entity.getContentType()); try (XContentParser parser = createParser(entityContentType.xContent(), entity.getContent())) { parsedUpdateRequest.fromXContent(parser); } diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java index 1ac5ff06d5624..0cfbd8c29fe43 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java @@ -69,8 +69,8 @@ private void assertMappingsEqual(CreateIndexRequest expected, CreateIndexRequest } else { assertNotNull(actual.mappings()); try ( - XContentParser expectedJson = createParser(expected.mappingsXContentType().xContent(), expected.mappings()); - XContentParser actualJson = createParser(actual.mappingsXContentType().xContent(), actual.mappings()) + XContentParser expectedJson = createParser(expected.mappingsMediaType().xContent(), expected.mappings()); + XContentParser actualJson = createParser(actual.mappingsMediaType().xContent(), actual.mappings()) ) { assertEquals(expectedJson.map(), actualJson.map()); } catch (IOException e) { diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java index 24795204c6dc8..1d7321bf2c6de 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java @@ -53,6 +53,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import java.io.ByteArrayInputStream; import java.io.EOFException; @@ -344,6 +346,10 @@ public BigInteger readBigInteger() throws IOException { return new BigInteger(readString()); } + public MediaType readMediaType() throws IOException { + return MediaTypeParserRegistry.fromMediaType(readString()); + } + @Nullable public Text readOptionalText() throws IOException { int length = readInt(); diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java index 82d16315f6527..c1409e551e47d 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java @@ -32,12 +32,16 @@ package org.opensearch.core.xcontent; +import org.opensearch.core.common.io.stream.Writeable; + +import java.util.Locale; + /** * Abstracts a Media Type and a format parameter. * Media types are used as values on Content-Type and Accept headers * format is an URL parameter, specifies response media type. */ -public interface MediaType { +public interface MediaType extends Writeable { /** * Returns a type part of a MediaType * i.e. application for application/json @@ -70,4 +74,41 @@ default String mediaType() { } String mediaTypeWithoutParameters(); + + /** + * Accepts a format string, which is most of the time is equivalent to {@link MediaType#subtype()} + * and attempts to match the value to an {@link MediaType}. + * The comparisons are done in lower case format. + * This method will return {@code null} if no match is found + */ + static MediaType fromFormat(String mediaType) { + return MediaTypeParserRegistry.fromFormat(mediaType); + } + + /** + * Attempts to match the given media type with the known {@link MediaType} values. This match is done in a case-insensitive manner. + * The provided media type can optionally has parameters. + * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers. + * This method will return {@code null} if no match is found + */ + static MediaType fromMediaType(String mediaTypeHeaderValue) { + mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue); + return MediaTypeParserRegistry.fromMediaType(mediaTypeHeaderValue); + } + + /** + * Clients compatible with ES 7.x might start sending media types with versioned media type + * in a form of application/vnd.elasticsearch+json;compatible-with=7. + * This has to be removed in order to be used in 7.x server. + * The same client connecting using that media type will be able to communicate with ES 8 thanks to compatible API. + * @param mediaType - a media type used on Content-Type header, might contain versioned media type. + * + * @return a media type string without + */ + private static String removeVersionInMediaType(String mediaType) { + if (mediaType != null && (mediaType = mediaType.toLowerCase(Locale.ROOT)).contains("vnd.opensearch")) { + return mediaType.replaceAll("vnd.opensearch\\+", "").replaceAll("\\s*;\\s*compatible-with=\\d+", ""); + } + return mediaType; + } } diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParserRegistry.java similarity index 53% rename from libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java rename to libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParserRegistry.java index cce95e7b4d6b1..62a26b4458b09 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParserRegistry.java @@ -41,40 +41,52 @@ * * @opensearch.internal */ -public class MediaTypeParser { - private final Map formatToMediaType; - private final Map typeWithSubtypeToMediaType; - - public MediaTypeParser(T[] acceptedMediaTypes) { - this(acceptedMediaTypes, Map.of()); - } - - public MediaTypeParser(T[] acceptedMediaTypes, Map additionalMediaTypes) { - final int size = acceptedMediaTypes.length + additionalMediaTypes.size(); - Map formatMap = new HashMap<>(size); - Map typeMap = new HashMap<>(size); - for (T mediaType : acceptedMediaTypes) { +public final class MediaTypeParserRegistry { + private static Map formatToMediaType = Map.of(); + private static Map typeWithSubtypeToMediaType = Map.of(); + + // Default mediaType singleton + private static MediaType DEFAULT_MEDIA_TYPE; + + public static void register(MediaType[] acceptedMediaTypes, Map additionalMediaTypes) { + // ensures the map is not overwritten: + Map typeMap = new HashMap<>(typeWithSubtypeToMediaType); + Map formatMap = new HashMap<>(formatToMediaType); + for (MediaType mediaType : acceptedMediaTypes) { + if (formatMap.containsKey(mediaType.format())) { + throw new IllegalArgumentException("unable to register mediaType: [" + mediaType.format() + "]. Type already exists."); + } typeMap.put(mediaType.typeWithSubtype(), mediaType); formatMap.put(mediaType.format(), mediaType); } - for (Map.Entry entry : additionalMediaTypes.entrySet()) { - String typeWithSubtype = entry.getKey(); - T mediaType = entry.getValue(); + for (Map.Entry entry : additionalMediaTypes.entrySet()) { + String typeWithSubtype = entry.getKey().toLowerCase(Locale.ROOT); + if (typeMap.containsKey(typeWithSubtype)) { + throw new IllegalArgumentException( + "unable to register mediaType: [" + + entry.getKey() + + "]. " + + "Type already exists and is mapped to: [." + + entry.getValue().format() + + "]" + ); + } - typeMap.put(typeWithSubtype.toLowerCase(Locale.ROOT), mediaType); - formatMap.put(mediaType.format(), mediaType); + MediaType mediaType = entry.getValue(); + typeMap.put(typeWithSubtype, mediaType); + formatMap.putIfAbsent(mediaType.format(), mediaType); // ignore if the additional type mapping already exists } - this.formatToMediaType = Map.copyOf(formatMap); - this.typeWithSubtypeToMediaType = Map.copyOf(typeMap); + formatToMediaType = Map.copyOf(formatMap); + typeWithSubtypeToMediaType = Map.copyOf(typeMap); } - public T fromMediaType(String mediaType) { + public static MediaType fromMediaType(String mediaType) { ParsedMediaType parsedMediaType = parseMediaType(mediaType); return parsedMediaType != null ? parsedMediaType.getMediaType() : null; } - public T fromFormat(String format) { + public static MediaType fromFormat(String format) { if (format == null) { return null; } @@ -86,7 +98,7 @@ public T fromFormat(String format) { * @param headerValue a header value from Accept or Content-Type * @return a parsed media-type */ - public ParsedMediaType parseMediaType(String headerValue) { + public static ParsedMediaType parseMediaType(String headerValue) { if (headerValue != null) { String[] split = headerValue.toLowerCase(Locale.ROOT).split(";"); @@ -94,8 +106,8 @@ public ParsedMediaType parseMediaType(String headerValue) { if (typeSubtype.length == 2) { String type = typeSubtype[0]; String subtype = typeSubtype[1]; - T xContentType = typeWithSubtypeToMediaType.get(type + "/" + subtype); - if (xContentType != null) { + MediaType mediaType = typeWithSubtypeToMediaType.get(type + "/" + subtype); + if (mediaType != null) { Map parameters = new HashMap<>(); for (int i = 1; i < split.length; i++) { // spaces are allowed between parameters, but not between '=' sign @@ -105,7 +117,7 @@ public ParsedMediaType parseMediaType(String headerValue) { } parameters.put(keyValueParam[0], keyValueParam[1]); } - return new ParsedMediaType(xContentType, parameters); + return new ParsedMediaType(mediaType, parameters); } } @@ -113,23 +125,23 @@ public ParsedMediaType parseMediaType(String headerValue) { return null; } - private boolean hasSpaces(String s) { + private static boolean hasSpaces(String s) { return s.trim().equals(s) == false; } /** * A media type object that contains all the information provided on a Content-Type or Accept header */ - public class ParsedMediaType { + public static class ParsedMediaType { private final Map parameters; - private final T mediaType; + private final MediaType mediaType; - public ParsedMediaType(T mediaType, Map parameters) { + public ParsedMediaType(MediaType mediaType, Map parameters) { this.parameters = parameters; this.mediaType = mediaType; } - public T getMediaType() { + public MediaType getMediaType() { return mediaType; } @@ -137,4 +149,18 @@ public Map getParameters() { return parameters; } } + + public static void setDefaultMediaType(final MediaType mediaType) { + if (DEFAULT_MEDIA_TYPE != null) { + throw new RuntimeException( + "unable to reset the default media type from current default [" + DEFAULT_MEDIA_TYPE + "] to [" + mediaType + "]" + ); + } else { + DEFAULT_MEDIA_TYPE = mediaType; + } + } + + public static MediaType getDefaultMediaType() { + return DEFAULT_MEDIA_TYPE; + } } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java index 94ac264546dbb..76a2046dd768a 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java @@ -113,7 +113,7 @@ public static XContentBuilder cborBuilder(OutputStream os) throws IOException { /** * Constructs a xcontent builder that will output the result into the provided output stream. */ - public static XContentBuilder contentBuilder(XContentType type, OutputStream outputStream) throws IOException { + public static XContentBuilder contentBuilder(MediaType type, OutputStream outputStream) throws IOException { if (type == XContentType.JSON) { return jsonBuilder(outputStream); } else if (type == XContentType.SMILE) { @@ -131,7 +131,7 @@ public static XContentBuilder contentBuilder(XContentType type, OutputStream out */ public static XContentBuilder contentBuilder(MediaType type) throws IOException { if (type instanceof XContentType) { - return contentBuilder(XContentType.fromMediaType(type)); + return contentBuilder((XContentType) (type)); } throw new IllegalArgumentException("Content type [" + type.getClass().getName() + "] not supported"); } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java index b3a8f5e27635d..023caa49e1f39 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java @@ -36,11 +36,12 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; import org.opensearch.common.xcontent.yaml.YamlXContent; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParser; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import org.opensearch.core.xcontent.XContent; -import java.util.Locale; +import java.io.IOException; import java.util.Map; /** @@ -130,53 +131,9 @@ public XContent xContent() { } }; - /** a parser of media types */ - private static final MediaTypeParser MEDIA_TYPE_PARSER = new MediaTypeParser<>( - XContentType.values(), - Map.of("application/*", JSON, "application/x-ndjson", JSON) - ); - - /** gets the {@link MediaTypeParser} singleton for use outside class */ - @SuppressWarnings("rawtypes") - public static MediaTypeParser getMediaTypeParser() { - return MEDIA_TYPE_PARSER; - } - - /** - * Accepts a format string, which is most of the time is equivalent to {@link XContentType#subtype()} - * and attempts to match the value to an {@link XContentType}. - * The comparisons are done in lower case format. - * This method will return {@code null} if no match is found - */ - public static XContentType fromFormat(String mediaType) { - return MEDIA_TYPE_PARSER.fromFormat(mediaType); - } - - /** - * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner. - * The provided media type can optionally has parameters. - * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers. - * This method will return {@code null} if no match is found - */ - public static XContentType fromMediaType(String mediaTypeHeaderValue) { - mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue); - return MEDIA_TYPE_PARSER.fromMediaType(mediaTypeHeaderValue); - } - - /** - * Clients compatible with ES 7.x might start sending media types with versioned media type - * in a form of application/vnd.elasticsearch+json;compatible-with=7. - * This has to be removed in order to be used in 7.x server. - * The same client connecting using that media type will be able to communicate with ES 8 thanks to compatible API. - * @param mediaType - a media type used on Content-Type header, might contain versioned media type. - * - * @return a media type string without - */ - private static String removeVersionInMediaType(String mediaType) { - if (mediaType != null && (mediaType = mediaType.toLowerCase(Locale.ROOT)).contains("vnd.opensearch")) { - return mediaType.replaceAll("vnd.opensearch\\+", "").replaceAll("\\s*;\\s*compatible-with=\\d+", ""); - } - return mediaType; + static { + /** a parser of media types */ + MediaTypeParserRegistry.register(XContentType.values(), Map.of("application/*", JSON, "application/x-ndjson", JSON)); } private int index; @@ -199,12 +156,8 @@ public String format() { return subtype(); } - /** Converts from a {@link MediaType} to an explicit {@link XContentType} */ - public static XContentType fromMediaType(MediaType mediaType) { - if (mediaType instanceof XContentType) { - return (XContentType) mediaType; - } else { - return mediaType != null ? MEDIA_TYPE_PARSER.fromMediaType(mediaType.mediaTypeWithoutParameters()) : null; - } + @Override + public void writeTo(StreamOutput output) throws IOException { + output.writeString(this.mediaType()); } } diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java index f5616f2159f77..15492b7351984 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.xcontent; -import org.opensearch.core.xcontent.MediaTypeParser; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import org.opensearch.test.OpenSearchTestCase; import java.util.Collections; @@ -44,42 +44,42 @@ public class MediaTypeParserTests extends OpenSearchTestCase { - @SuppressWarnings("unchecked") - MediaTypeParser mediaTypeParser = XContentType.getMediaTypeParser(); - public void testJsonWithParameters() throws Exception { String mediaType = "application/json"; - assertThat(mediaTypeParser.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap())); - assertThat(mediaTypeParser.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8"))); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap())); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap())); + assertThat( + MediaTypeParserRegistry.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), + equalTo(Map.of("charset", "utf-8")) + ); assertThat( - mediaTypeParser.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(), + MediaTypeParserRegistry.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); } public void testWhiteSpaceInTypeSubtype() { String mediaType = " application/json "; - assertThat(mediaTypeParser.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON)); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON)); assertThat( - mediaTypeParser.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(), + MediaTypeParserRegistry.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); assertThat( - mediaTypeParser.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(), + MediaTypeParserRegistry.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); mediaType = " application / json "; - assertThat(mediaTypeParser.parseMediaType(mediaType), is(nullValue())); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType), is(nullValue())); } public void testInvalidParameters() { String mediaType = "application/json"; - assertThat(mediaTypeParser.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue())); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; key = value"), is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; key="), is(nullValue())); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + "; key = value"), is(nullValue())); + assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + "; key="), is(nullValue())); } } diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java index faae24083eeca..9a2c8c1f0aa55 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java @@ -65,9 +65,9 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -178,11 +178,11 @@ static class ContextSetup implements Writeable, ToXContentObject { private final BytesReference document; private final QueryBuilder query; - private XContentType xContentType; + private MediaType mediaType; static ContextSetup parse(XContentParser parser, Void context) throws IOException { ContextSetup contextSetup = PARSER.parse(parser, null); - contextSetup.setXContentType(XContentType.fromMediaType(parser.contentType())); + contextSetup.setXContentType(parser.contentType()); return contextSetup; } @@ -195,9 +195,9 @@ static ContextSetup parse(XContentParser parser, Void context) throws IOExceptio ContextSetup(StreamInput in) throws IOException { index = in.readOptionalString(); document = in.readOptionalBytesReference(); - String xContentType = in.readOptionalString(); - if (xContentType != null) { - this.xContentType = XContentType.fromMediaType(xContentType); + String mediaType = in.readOptionalString(); + if (mediaType != null) { + this.mediaType = MediaType.fromMediaType(mediaType); } query = in.readOptionalNamedWriteable(QueryBuilder.class); } @@ -214,12 +214,12 @@ public QueryBuilder getQuery() { return query; } - public XContentType getXContentType() { - return xContentType; + public MediaType getXContentType() { + return mediaType; } - public void setXContentType(XContentType xContentType) { - this.xContentType = xContentType; + public void setXContentType(MediaType mediaType) { + this.mediaType = mediaType; } @Override @@ -230,19 +230,19 @@ public boolean equals(Object o) { return Objects.equals(index, that.index) && Objects.equals(document, that.document) && Objects.equals(query, that.query) - && Objects.equals(xContentType, that.xContentType); + && Objects.equals(mediaType, that.mediaType); } @Override public int hashCode() { - return Objects.hash(index, document, query, xContentType); + return Objects.hash(index, document, query, mediaType); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(index); out.writeOptionalBytesReference(document); - out.writeOptionalString(xContentType != null ? xContentType.mediaTypeWithoutParameters() : null); + out.writeOptionalString(mediaType != null ? mediaType.mediaTypeWithoutParameters() : null); out.writeOptionalNamedWriteable(query); } @@ -257,7 +257,7 @@ public String toString() { + ", query=" + query + ", xContentType=" - + xContentType + + mediaType + '}'; } @@ -275,7 +275,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, document, - xContentType + mediaType ) ) { builder.generator().copyCurrentStructure(parser); @@ -594,8 +594,8 @@ private static Response prepareRamIndex( try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(defaultAnalyzer))) { String index = indexService.index().getName(); BytesReference document = request.contextSetup.document; - XContentType xContentType = request.contextSetup.xContentType; - SourceToParse sourceToParse = new SourceToParse(index, "_id", document, xContentType); + MediaType mediaType = request.contextSetup.mediaType; + SourceToParse sourceToParse = new SourceToParse(index, "_id", document, mediaType); ParsedDocument parsedDocument = indexService.mapperService().documentMapper().parse(sourceToParse); indexWriter.addDocuments(parsedDocument.docs()); try (IndexReader indexReader = DirectoryReader.open(indexWriter)) { diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java index ba974d248ac9b..cb07e593f8155 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java @@ -61,10 +61,10 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.index.reindex.remote.RemoteScrollableHitSource; @@ -378,27 +378,24 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) index.id(doc.getId()); // the source xcontent type and destination could be different - final XContentType sourceXContentType = doc.getXContentType(); - final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType(); - if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) { + final MediaType sourceMediaType = doc.getMediaType(); + final MediaType mainRequestMediaType = mainRequest.getDestination().getContentType(); + if (mainRequestMediaType != null && doc.getMediaType() != mainRequestMediaType) { // we need to convert try ( InputStream stream = doc.getSource().streamInput(); - XContentParser parser = sourceXContentType.xContent() + XContentParser parser = sourceMediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream); - XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent()) + XContentBuilder builder = XContentBuilder.builder(mainRequestMediaType.xContent()) ) { parser.nextToken(); builder.copyCurrentStructure(parser); index.source(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { - throw new UncheckedIOException( - "failed to convert hit from " + sourceXContentType + " to " + mainRequestXContentType, - e - ); + throw new UncheckedIOException("failed to convert hit from " + sourceMediaType + " to " + mainRequestMediaType, e); } } else { - index.source(doc.getSource(), doc.getXContentType()); + index.source(doc.getSource(), doc.getMediaType()); } /* diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java index 5297723ab3687..1ea9ec5fb7beb 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java @@ -138,7 +138,7 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) IndexRequest index = new IndexRequest(); index.index(doc.getIndex()); index.id(doc.getId()); - index.source(doc.getSource(), doc.getXContentType()); + index.source(doc.getSource(), doc.getMediaType()); index.setIfSeqNo(doc.getSeqNo()); index.setIfPrimaryTerm(doc.getPrimaryTerm()); index.setPipeline(mainRequest.getPipeline()); diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java index 3d606b6b77c44..aaca04641d76a 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java @@ -41,12 +41,12 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.reindex.ScrollableHitSource.BasicHit; import org.opensearch.index.reindex.ScrollableHitSource.Hit; import org.opensearch.index.reindex.ScrollableHitSource.Response; @@ -72,7 +72,7 @@ private RemoteResponseParsers() {} /** * Parser for an individual {@code hit} element. */ - public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { + public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { int i = 0; String index = (String) a[i++]; String id = (String) a[i++]; @@ -106,7 +106,7 @@ private RemoteResponseParsers() {} class Fields { String routing; } - ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); + ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); HIT_PARSER.declareObject((hit, fields) -> { hit.setRouting(fields.routing); }, fieldsParser, new ParseField("fields")); fieldsParser.declareString((fields, routing) -> fields.routing = routing, routingField); fieldsParser.declareLong((fields, ttl) -> {}, ttlField); // ignore ttls since they have been removed @@ -116,7 +116,7 @@ class Fields { /** * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List)]}. */ - public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); + public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); static { HITS_PARSER.declareField(constructorArg(), (p, c) -> { if (p.currentToken() == XContentParser.Token.START_OBJECT) { @@ -184,7 +184,7 @@ class Fields { SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures")); } - public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>( "search_response", true, a -> { @@ -296,13 +296,13 @@ public void setCausedBy(Throwable causedBy) { /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. */ - public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( "/", true, a -> (Version) a[0] ); static { - ConstructingObjectParser versionParser = new ConstructingObjectParser<>( + ConstructingObjectParser versionParser = new ConstructingObjectParser<>( "version", true, a -> a[0] == null diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java index 39d3adcd82ad5..3c305c2a8239d 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -55,10 +55,10 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.reindex.RejectAwareActionListener; import org.opensearch.index.reindex.ScrollableHitSource; import org.opensearch.core.rest.RestStatus; @@ -182,7 +182,7 @@ protected void cleanup(Runnable onCompletion) { private void execute( Request request, - BiFunction parser, + BiFunction parser, RejectAwareActionListener listener ) { // Preserve the thread context so headers survive after the call @@ -198,12 +198,12 @@ public void onSuccess(org.opensearch.client.Response response) { try { HttpEntity responseEntity = response.getEntity(); InputStream content = responseEntity.getContent(); - XContentType xContentType = null; + MediaType mediaType = null; if (responseEntity.getContentType() != null) { final String mimeType = ContentType.parse(responseEntity.getContentType()).getMimeType(); - xContentType = XContentType.fromMediaType(mimeType); + mediaType = MediaType.fromMediaType(mimeType); } - if (xContentType == null) { + if (mediaType == null) { try { logger.debug("Response didn't include Content-Type: " + bodyMessage(response.getEntity())); throw new OpenSearchException( @@ -217,10 +217,10 @@ public void onSuccess(org.opensearch.client.Response response) { } // EMPTY is safe here because we don't call namedObject try ( - XContentParser xContentParser = xContentType.xContent() + XContentParser xContentParser = mediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, content) ) { - parsedResponse = parser.apply(xContentParser, xContentType); + parsedResponse = parser.apply(xContentParser, mediaType); } catch (XContentParseException e) { /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it * is totally wrong and we're probably not talking to Elasticsearch. */ diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java index 2946544a8f901..651ccd47df7bd 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java @@ -74,7 +74,7 @@ public void testPipelineQueryParameterIsError() throws IOException { body.endObject(); } body.endObject(); - request.withContent(BytesReference.bytes(body), XContentType.fromMediaType(body.contentType())); + request.withContent(BytesReference.bytes(body), body.contentType()); } request.withParams(singletonMap("pipeline", "doesn't matter")); Exception e = expectThrows( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java index 7f306b3ee550f..bb14ed1ea5578 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java @@ -406,9 +406,7 @@ public void testBulk() throws Exception { .endObject(); client().admin().indices().prepareCreate("countries").setSettings(settings).setMapping(xContentBuilder).get(); - BulkResponse bulk = client().prepareBulk() - .add(bulkAction, 0, bulkAction.length, null, XContentType.fromMediaType(xContentBuilder.contentType())) - .get(); + BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, xContentBuilder.contentType()).get(); for (BulkItemResponse item : bulk.getItems()) { assertFalse("unable to index data", item.isFailed()); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index ecfcd6310dd0a..761373a001ffe 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -32,12 +32,14 @@ package org.opensearch.action.admin.cluster.storedscripts; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; @@ -59,14 +61,18 @@ public class PutStoredScriptRequest extends AcknowledgedRequest indexRequestConsumer, Consumer updateRequestConsumer, Consumer deleteRequestConsumer ) throws IOException { - XContent xContent = xContentType.xContent(); + XContent xContent = mediaType.xContent(); int line = 0; int from = 0; byte marker = xContent.streamSeparator(); @@ -312,7 +313,7 @@ public void parse( .setPipeline(pipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType) .setRequireAlias(requireAlias) ); } else { @@ -325,7 +326,7 @@ public void parse( .setPipeline(pipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType) .setRequireAlias(requireAlias) ); } @@ -339,7 +340,7 @@ public void parse( .setPipeline(pipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType) .setRequireAlias(requireAlias) ); } else if ("update".equals(action)) { @@ -358,7 +359,7 @@ public void parse( .routing(routing); try ( XContentParser sliceParser = createParser( - sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), + sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), xContent ) ) { diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java index 0b675fd9125fa..140c1320daa91 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java @@ -78,8 +78,8 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.lease.Releasable; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationPressureService; @@ -752,7 +752,7 @@ static BulkItemResponse processUpdateResponse( if (updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) { final BytesReference indexSourceAsBytes = updateIndexRequest.source(); - final Tuple> sourceAndContent = XContentHelper.convertToMap( + final Tuple> sourceAndContent = XContentHelper.convertToMap( indexSourceAsBytes, true, updateIndexRequest.getContentType() diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequest.java b/server/src/main/java/org/opensearch/action/index/IndexRequest.java index 86ba13dedfa06..d686f0b460634 100644 --- a/server/src/main/java/org/opensearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/opensearch/action/index/IndexRequest.java @@ -78,10 +78,10 @@ * created using {@link org.opensearch.client.Requests#indexRequest(String)}. * * The index requires the {@link #index()}, {@link #id(String)} and - * {@link #source(byte[], XContentType)} to be set. + * {@link #source(byte[], MediaType)} to be set. * - * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}), - * its string form ({@link #source(String, XContentType)}) or using a {@link XContentBuilder} + * The source (content to index) can be set in its bytes form using ({@link #source(byte[], MediaType)}), + * its string form ({@link #source(String, MediaType)}) or using a {@link XContentBuilder} * ({@link #source(XContentBuilder)}). * * If the {@link #id(String)} is not set, it will be automatically generated. @@ -116,7 +116,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; - private XContentType contentType; + private MediaType contentType; private String pipeline; private String finalPipeline; @@ -159,7 +159,11 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); if (in.readBoolean()) { - contentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + contentType = in.readMediaType(); + } else { + contentType = in.readEnum(XContentType.class); + } } else { contentType = null; } @@ -174,7 +178,7 @@ public IndexRequest() { /** * Constructs a new index request against the specific index. The - * {@link #source(byte[], XContentType)} must be set. + * {@link #source(byte[], MediaType)} must be set. */ public IndexRequest(String index) { super(NO_SHARD_ID); @@ -249,7 +253,7 @@ public IndicesOptions indicesOptions() { * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the * source at index time */ - public XContentType getContentType() { + public MediaType getContentType() { return contentType; } @@ -372,7 +376,7 @@ public IndexRequest source(Map source) throws OpenSearchGenerationExc * * @param source The map to index */ - public IndexRequest source(Map source, XContentType contentType) throws OpenSearchGenerationException { + public IndexRequest source(Map source, MediaType contentType) throws OpenSearchGenerationException { try { XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.map(source); @@ -386,10 +390,10 @@ public IndexRequest source(Map source, XContentType contentType) thro * Sets the document source to index. * * Note, its preferable to either set it using {@link #source(XContentBuilder)} - * or using the {@link #source(byte[], XContentType)}. + * or using the {@link #source(byte[], MediaType)}. */ - public IndexRequest source(String source, XContentType xContentType) { - return source(new BytesArray(source), xContentType); + public IndexRequest source(String source, MediaType mediaType) { + return source(new BytesArray(source), mediaType); } /** @@ -419,7 +423,7 @@ public IndexRequest source(Object... source) { * valid String representation. *

    */ - public IndexRequest source(XContentType xContentType, Object... source) { + public IndexRequest source(MediaType mediaType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } @@ -430,7 +434,7 @@ public IndexRequest source(XContentType xContentType, Object... source) { ); } try { - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = XContentFactory.contentBuilder(mediaType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); @@ -447,27 +451,28 @@ public IndexRequest source(XContentType xContentType, Object... source) { */ public IndexRequest source(BytesReference source, MediaType mediaType) { this.source = Objects.requireNonNull(source); - this.contentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType)); + this.contentType = Objects.requireNonNull(mediaType); return this; } /** * Sets the document to index in bytes form. */ - public IndexRequest source(byte[] source, XContentType xContentType) { - return source(source, 0, source.length, xContentType); + public IndexRequest source(byte[] source, MediaType mediaType) { + return source(source, 0, source.length, mediaType); } /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). * - * @param source The source to index - * @param offset The offset in the byte array - * @param length The length of the data + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + * @param mediaType The data format over the wire */ - public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { - return source(new BytesArray(source, offset, length), xContentType); + public IndexRequest source(byte[] source, int offset, int length, MediaType mediaType) { + return source(new BytesArray(source, offset, length), mediaType); } /** @@ -665,7 +670,11 @@ private void writeBody(StreamOutput out) throws IOException { out.writeLong(autoGeneratedTimestamp); if (contentType != null) { out.writeBoolean(true); - out.writeEnum(contentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + contentType.writeTo(out); + } else { + out.writeEnum((XContentType) contentType); + } } else { out.writeBoolean(false); } diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java index a3067fe0ba160..7a88f817c70bf 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java @@ -32,12 +32,13 @@ package org.opensearch.action.ingest; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -54,19 +55,7 @@ public class PutPipelineRequest extends AcknowledgedRequest private String id; private BytesReference source; - private XContentType xContentType; - - /** - * Create a new pipeline request with the id and source along with the content type of the source - * - * @deprecated use {@link #PutPipelineRequest(String, BytesReference, MediaType)} instead - */ - @Deprecated - public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { - this.id = Objects.requireNonNull(id); - this.source = Objects.requireNonNull(source); - this.xContentType = Objects.requireNonNull(xContentType); - } + private MediaType mediaType; /** * Create a new pipeline request with the id and source along with the content type of the source @@ -74,17 +63,18 @@ public PutPipelineRequest(String id, BytesReference source, XContentType xConten public PutPipelineRequest(String id, BytesReference source, MediaType mediaType) { this.id = Objects.requireNonNull(id); this.source = Objects.requireNonNull(source); - if (mediaType instanceof XContentType == false) { - throw new IllegalArgumentException("PutPipelineRequest found unsupported media type [" + mediaType.getClass().getName() + "]"); - } - this.xContentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public PutPipelineRequest(StreamInput in) throws IOException { super(in); id = in.readString(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } PutPipelineRequest() {} @@ -102,8 +92,8 @@ public BytesReference getSource() { return source; } - public XContentType getXContentType() { - return xContentType; + public MediaType getMediaType() { + return mediaType; } @Override @@ -111,13 +101,17 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (source != null) { - builder.rawValue(source.streamInput(), xContentType); + builder.rawValue(source.streamInput(), mediaType); } else { builder.startObject().endObject(); } diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java index 31f71e5720daa..1ac441a1afe64 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java @@ -32,15 +32,17 @@ package org.opensearch.action.ingest; +import org.opensearch.Version; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.ingest.IngestDocument; @@ -66,14 +68,14 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent private String id; private boolean verbose; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; /** * Creates a new request with the given source and its content type */ - public SimulatePipelineRequest(BytesReference source, XContentType xContentType) { + public SimulatePipelineRequest(BytesReference source, MediaType mediaType) { this.source = Objects.requireNonNull(source); - this.xContentType = Objects.requireNonNull(xContentType); + this.mediaType = Objects.requireNonNull(mediaType); } SimulatePipelineRequest() {} @@ -83,7 +85,11 @@ public SimulatePipelineRequest(BytesReference source, XContentType xContentType) id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } @Override @@ -111,8 +117,8 @@ public BytesReference getSource() { return source; } - public XContentType getXContentType() { - return xContentType; + public MediaType getXContentType() { + return mediaType; } @Override @@ -121,12 +127,16 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.rawValue(source.streamInput(), xContentType); + builder.rawValue(source.streamInput(), mediaType); return builder; } diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java index 822d1f2320c9b..d32aab0c8a561 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java @@ -8,6 +8,7 @@ package org.opensearch.action.search; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.core.common.bytes.BytesReference; @@ -29,7 +30,7 @@ public class PutSearchPipelineRequest extends AcknowledgedRequest implements ToXContentObject { private String id; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; public PutSearchPipelineRequest(String id, BytesReference source, MediaType mediaType) { this.id = Objects.requireNonNull(id); @@ -39,14 +40,18 @@ public PutSearchPipelineRequest(String id, BytesReference source, MediaType medi PutSearchPipelineRequest.class.getSimpleName() + " found unsupported media type [" + mediaType.getClass().getName() + "]" ); } - this.xContentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public PutSearchPipelineRequest(StreamInput in) throws IOException { super(in); id = in.readString(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } @Override @@ -62,8 +67,8 @@ public BytesReference getSource() { return source; } - public XContentType getXContentType() { - return xContentType; + public MediaType getMediaType() { + return mediaType; } @Override @@ -71,13 +76,17 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (source != null) { - builder.rawValue(source.streamInput(), xContentType); + builder.rawValue(source.streamInput(), mediaType); } else { builder.startObject().endObject(); } diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java index d33882fc1d53d..71200b05d70ad 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java @@ -40,6 +40,7 @@ import org.opensearch.action.get.MultiGetRequest; import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.common.Nullable; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -51,7 +52,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; @@ -94,7 +94,7 @@ public class TermVectorsRequest extends SingleShardRequest i private BytesReference doc; - private XContentType xContentType; + private MediaType mediaType; private String routing; @@ -186,7 +186,11 @@ public TermVectorsRequest() {} if (in.readBoolean()) { doc = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } routing = in.readOptionalString(); preference = in.readOptionalString(); @@ -235,7 +239,7 @@ public TermVectorsRequest(TermVectorsRequest other) { this.id = other.id(); if (other.doc != null) { this.doc = new BytesArray(other.doc().toBytesRef(), true); - this.xContentType = other.xContentType; + this.mediaType = other.mediaType; } this.flagsEnum = other.getFlags().clone(); this.preference = other.preference(); @@ -285,8 +289,8 @@ public BytesReference doc() { return doc; } - public XContentType xContentType() { - return xContentType; + public MediaType xContentType() { + return mediaType; } /** @@ -308,13 +312,13 @@ public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { /** * Sets an artificial document from which term vectors are requested for. */ - public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, MediaType xContentType) { + public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, MediaType mediaType) { // assign a random id to this artificial document, for routing if (generateRandomId) { this.id(String.valueOf(randomInt.getAndAdd(1))); } this.doc = doc; - this.xContentType = XContentType.fromMediaType(xContentType); + this.mediaType = mediaType; return this; } @@ -534,7 +538,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } out.writeOptionalString(routing); out.writeOptionalString(preference); diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java index d20d5e229a4c7..95433bc42c2d3 100644 --- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java @@ -61,7 +61,7 @@ import org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.engine.VersionConflictEngineException; @@ -238,7 +238,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< response.getResult() ); if (request.fetchSource() != null && request.fetchSource().fetchSource()) { - Tuple> sourceAndContent = XContentHelper.convertToMap( + Tuple> sourceAndContent = XContentHelper.convertToMap( upsertSourceBytes, true, upsertRequest.getContentType() diff --git a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java index e3aa4f1161ac7..a5568a838f21f 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java @@ -40,10 +40,11 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Requests; import org.opensearch.common.Nullable; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -352,7 +353,7 @@ public static GetResult extractGetResult( long primaryTerm, long version, final Map source, - XContentType sourceContentType, + MediaType sourceContentType, @Nullable final BytesReference sourceAsBytes ) { if (request.fetchSource() == null || request.fetchSource().fetchSource() == false) { diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java index 8f529ec855d9b..44454630ff24d 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java @@ -48,6 +48,7 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; @@ -922,13 +923,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("doc_as_upsert", docAsUpsert); } if (doc != null) { - XContentType xContentType = doc.getContentType(); + MediaType mediaType = doc.getContentType(); try ( XContentParser parser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, doc.source(), - xContentType + mediaType ) ) { builder.field("doc"); @@ -945,13 +946,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("script", script); } if (upsertRequest != null) { - XContentType xContentType = upsertRequest.getContentType(); + MediaType mediaType = upsertRequest.getContentType(); try ( XContentParser parser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, upsertRequest.source(), - xContentType + mediaType ) ) { builder.field("upsert"); diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java index cfe8008a4adca..ae1fb0724fd9e 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java @@ -497,6 +497,7 @@ public static void writeRawField(String field, BytesReference source, XContentTy * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned * by the {@link ToXContent#isFragment()} method returns. */ + @Deprecated public static BytesReference toXContent(ToXContent toXContent, XContentType xContentType, boolean humanReadable) throws IOException { return toXContent(toXContent, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); } diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java index 0992345735176..e6df6e964a31b 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java @@ -9,13 +9,15 @@ package org.opensearch.extensions.rest; import org.opensearch.OpenSearchParseException; +import org.opensearch.Version; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestRequest.Method; import org.opensearch.transport.TransportRequest; @@ -42,7 +44,7 @@ public class ExtensionRestRequest extends TransportRequest { private String path; private Map params; private Map> headers; - private XContentType xContentType = null; + private MediaType mediaType = null; private BytesReference content; // The owner of this request object // Will be replaced with PrincipalIdentifierToken class from feature/identity @@ -56,15 +58,15 @@ public class ExtensionRestRequest extends TransportRequest { /** * This object can be instantiated given method, uri, params, content and identifier * - * @param method of type {@link Method} - * @param uri the REST uri string (excluding the query) - * @param path the REST path - * @param params the REST params - * @param headers the REST headers - * @param xContentType the content type, or null for plain text or no content - * @param content the REST request content + * @param method of type {@link Method} + * @param uri the REST uri string (excluding the query) + * @param path the REST path + * @param params the REST params + * @param headers the REST headers + * @param mediaType the content type, or null for plain text or no content + * @param content the REST request content * @param principalIdentifier the owner of this request - * @param httpVersion the REST HTTP protocol version + * @param httpVersion the REST HTTP protocol version */ public ExtensionRestRequest( Method method, @@ -72,7 +74,7 @@ public ExtensionRestRequest( String path, Map params, Map> headers, - XContentType xContentType, + MediaType mediaType, BytesReference content, String principalIdentifier, HttpRequest.HttpVersion httpVersion @@ -82,7 +84,7 @@ public ExtensionRestRequest( this.path = path; this.params = params; this.headers = headers; - this.xContentType = xContentType; + this.mediaType = mediaType; this.content = content; this.principalIdentifierToken = principalIdentifier; this.httpVersion = httpVersion; @@ -102,7 +104,11 @@ public ExtensionRestRequest(StreamInput in) throws IOException { params = in.readMap(StreamInput::readString, StreamInput::readString); headers = in.readMap(StreamInput::readString, StreamInput::readStringList); if (in.readBoolean()) { - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } content = in.readBytesReference(); principalIdentifierToken = in.readString(); @@ -117,9 +123,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(path); out.writeMap(params, StreamOutput::writeString, StreamOutput::writeString); out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeStringCollection); - out.writeBoolean(xContentType != null); - if (xContentType != null) { - out.writeEnum(xContentType); + out.writeBoolean(mediaType != null); + if (mediaType != null) { + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } out.writeBytesReference(content); out.writeString(principalIdentifierToken); @@ -237,8 +247,8 @@ public Map> headers() { * * @return the content type of the {@link #content()}, or null if the context is plain text or if there is no content. */ - public XContentType getXContentType() { - return xContentType; + public MediaType getXContentType() { + return mediaType; } /** @@ -311,7 +321,7 @@ public String toString() { + ", headers=" + headers.toString() + ", xContentType=" - + xContentType + + mediaType + ", contentLength=" + content.length() + ", requester=" @@ -331,7 +341,7 @@ public boolean equals(Object obj) { && Objects.equals(path, that.path) && Objects.equals(params, that.params) && Objects.equals(headers, that.headers) - && Objects.equals(xContentType, that.xContentType) + && Objects.equals(mediaType, that.mediaType) && Objects.equals(content, that.content) && Objects.equals(principalIdentifierToken, that.principalIdentifierToken) && Objects.equals(httpVersion, that.httpVersion); @@ -339,6 +349,6 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(method, uri, path, params, headers, xContentType, content, principalIdentifierToken, httpVersion); + return Objects.hash(method, uri, path, params, headers, mediaType, content, principalIdentifierToken, httpVersion); } } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 2a3ad63379556..4b622b841a040 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -80,7 +80,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client Tuple> unreadExtensionTuple = XContentHelper.convertToMap( request.content(), false, - request.getXContentType().xContent().mediaType() + request.getMediaType().xContent().mediaType() ); Map extensionMap = unreadExtensionTuple.v2(); diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index e20a3c6b2814c..a96e87b0200cc 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -14,7 +14,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.rest.BaseRestHandler; @@ -181,7 +181,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC String uri = httpRequest.uri(); Map params = request.params(); Map> headers = request.getHeaders(); - XContentType contentType = request.getXContentType(); + MediaType contentType = request.getMediaType(); BytesReference content = request.content(); HttpRequest.HttpVersion httpVersion = httpRequest.protocolVersion(); diff --git a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java index 87605969feb6a..86c5d4542e5fa 100644 --- a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java +++ b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java @@ -244,7 +244,7 @@ private static Map prepareMap( return map; } try { - String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getMediaType()); String trim = Strings.cleanTruncate(source, maxSourceCharsToLog).trim(); StringBuilder sb = new StringBuilder(trim); StringBuilders.escapeJson(sb, 0); @@ -279,7 +279,7 @@ private static String message(Index index, ParsedDocument doc, long tookInNanos, return sb.toString(); } try { - String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getMediaType()); sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog).trim()).append("]"); } catch (IOException e) { sb.append(", source[_failed_to_convert_[").append(e.getMessage()).append("]]"); diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index c5ee5be491095..f276d6ee2e579 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -43,8 +43,8 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DynamicTemplate.XContentFieldType; @@ -77,14 +77,14 @@ final class DocumentParser { ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadataFieldsMappers) throws MapperParsingException { final Mapping mapping = docMapper.mapping(); final ParseContext.InternalParseContext context; - final XContentType xContentType = source.getXContentType(); + final MediaType mediaType = source.getMediaType(); try ( XContentParser parser = XContentHelper.createParser( docMapperParser.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, source.source(), - xContentType + mediaType ) ) { context = new ParseContext.InternalParseContext(indexSettings, docMapperParser, docMapper, source, parser); @@ -183,7 +183,7 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext. source.routing(), context.docs(), context.sourceToParse().source(), - context.sourceToParse().getXContentType(), + context.sourceToParse().getMediaType(), update ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java index 46f0352976408..1084c08cdf32c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java @@ -35,6 +35,7 @@ import org.apache.lucene.document.Field; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.mapper.ParseContext.Document; import org.opensearch.index.mapper.MapperService.MergeReason; @@ -57,7 +58,7 @@ public class ParsedDocument { private final List documents; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; private Mapping dynamicMappingsUpdate; @@ -68,7 +69,7 @@ public ParsedDocument( String routing, List documents, BytesReference source, - XContentType xContentType, + MediaType mediaType, Mapping dynamicMappingsUpdate ) { this.version = version; @@ -78,7 +79,7 @@ public ParsedDocument( this.documents = documents; this.source = source; this.dynamicMappingsUpdate = dynamicMappingsUpdate; - this.xContentType = xContentType; + this.mediaType = mediaType; } public String id() { @@ -122,13 +123,13 @@ public BytesReference source() { return this.source; } - public XContentType getXContentType() { - return this.xContentType; + public MediaType getMediaType() { + return this.mediaType; } public void setSource(BytesReference source, XContentType xContentType) { this.source = source; - this.xContentType = xContentType; + this.mediaType = xContentType; } /** diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java index 95e71f168c2ca..ad467d045b7e4 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java @@ -45,9 +45,9 @@ import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; @@ -202,7 +202,7 @@ public boolean isComplete() { @Override public void preParse(ParseContext context) throws IOException { BytesReference originalSource = context.sourceToParse().source(); - XContentType contentType = context.sourceToParse().getXContentType(); + MediaType contentType = context.sourceToParse().getMediaType(); final BytesReference adaptedSource = applyFilters(originalSource, contentType); if (adaptedSource != null) { @@ -219,15 +219,15 @@ public void preParse(ParseContext context) throws IOException { } @Nullable - public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable XContentType contentType) throws IOException { + public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable MediaType contentType) throws IOException { if (enabled && originalSource != null) { // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); + Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); Map filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); - XContentType actualContentType = mapTuple.v1(); + MediaType actualContentType = mapTuple.v1(); XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); builder.close(); return bStream.bytes(); diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java index 085e258c5501e..2b24a380c8bb6 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java @@ -37,7 +37,7 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; /** * Stores the document source @@ -54,20 +54,20 @@ public class SourceToParse { private final @Nullable String routing; - private final XContentType xContentType; + private final MediaType mediaType; - public SourceToParse(String index, String id, BytesReference source, XContentType xContentType, @Nullable String routing) { + public SourceToParse(String index, String id, BytesReference source, MediaType mediaType, @Nullable String routing) { this.index = Objects.requireNonNull(index); this.id = Objects.requireNonNull(id); // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef()); - this.xContentType = Objects.requireNonNull(xContentType); + this.mediaType = Objects.requireNonNull(mediaType); this.routing = routing; } - public SourceToParse(String index, String id, BytesReference source, XContentType xContentType) { - this(index, id, source, xContentType, null); + public SourceToParse(String index, String id, BytesReference source, MediaType mediaType) { + this(index, id, source, mediaType, null); } public BytesReference source() { @@ -86,8 +86,8 @@ public String id() { return this.routing; } - public XContentType getXContentType() { - return this.xContentType; + public MediaType getMediaType() { + return this.mediaType; } /** diff --git a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java index 36b74c95c3200..7287634ecfacb 100644 --- a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java @@ -58,6 +58,7 @@ import org.opensearch.common.lucene.search.MoreLikeThisQuery; import org.opensearch.common.lucene.search.XMoreLikeThis; import org.opensearch.common.lucene.uid.Versions; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -170,7 +171,7 @@ public static final class Item implements ToXContentObject, Writeable { private String index; private String id; private BytesReference doc; - private XContentType xContentType; + private MediaType mediaType; private String[] fields; private Map perFieldAnalyzer; private String routing; @@ -187,7 +188,7 @@ public Item() {} this.id = copy.id; this.routing = copy.routing; this.doc = copy.doc; - this.xContentType = copy.xContentType; + this.mediaType = copy.mediaType; this.fields = copy.fields; this.perFieldAnalyzer = copy.perFieldAnalyzer; this.version = copy.version; @@ -220,7 +221,7 @@ public Item(@Nullable String index, XContentBuilder doc) { } this.index = index; this.doc = BytesReference.bytes(doc); - this.xContentType = XContentType.fromMediaType(doc.contentType()); + this.mediaType = doc.contentType(); } /** @@ -234,7 +235,11 @@ public Item(@Nullable String index, XContentBuilder doc) { } if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } else { id = in.readString(); } @@ -255,7 +260,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } else { out.writeString(id); } @@ -331,8 +340,8 @@ public Item versionType(VersionType versionType) { return this; } - XContentType xContentType() { - return xContentType; + MediaType mediaType() { + return mediaType; } /** @@ -351,7 +360,7 @@ TermVectorsRequest toTermVectorsRequest() { .termStatistics(false); // for artificial docs to make sure that the id has changed in the item too if (doc != null) { - termVectorsRequest.doc(doc, true, xContentType); + termVectorsRequest.doc(doc, true, mediaType); this.id = termVectorsRequest.id(); } return termVectorsRequest; @@ -373,7 +382,7 @@ public static Item parse(XContentParser parser, Item item) throws IOException { item.id = parser.text(); } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) { item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser)); - item.xContentType = XContentType.JSON; + item.mediaType = XContentType.JSON; } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { if (token == XContentParser.Token.START_ARRAY) { List fields = new ArrayList<>(); @@ -419,7 +428,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (this.doc != null) { try (InputStream stream = this.doc.streamInput()) { - builder.rawField(DOC.getPreferredName(), stream, xContentType); + builder.rawField(DOC.getPreferredName(), stream, mediaType); } } if (this.fields != null) { diff --git a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java index 998427c27c1f6..b3ed0079ff4c2 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java @@ -50,7 +50,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.search.SearchHit; import org.opensearch.threadpool.ThreadPool; @@ -209,7 +209,7 @@ public BytesReference getSource() { } @Override - public XContentType getXContentType() { + public MediaType getMediaType() { return XContentHelper.xContentType(source); } diff --git a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java index a4047507fcde1..47e882fc04127 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java @@ -46,6 +46,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; @@ -296,7 +297,7 @@ public interface Hit { * The content type of the hit source. Returns null if the source didn't come back from the search. */ @Nullable - XContentType getXContentType(); + MediaType getMediaType(); /** * The routing on the hit if there is any or null if there isn't. @@ -316,7 +317,7 @@ public static class BasicHit implements Hit { private final long version; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; private String routing; private long seqNo; private long primaryTerm; @@ -358,13 +359,13 @@ public BytesReference getSource() { } @Override - public XContentType getXContentType() { - return xContentType; + public MediaType getMediaType() { + return mediaType; } - public BasicHit setSource(BytesReference source, XContentType xContentType) { + public BasicHit setSource(BytesReference source, MediaType mediaType) { this.source = source; - this.xContentType = xContentType; + this.mediaType = mediaType; return this; } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index fbb2df7f9640a..0d8a54147d65e 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -43,7 +43,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.Term; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ReferenceManager; @@ -948,7 +947,7 @@ public Engine.IndexResult applyIndexOperationOnReplica( if (indexSettings.isSegRepEnabled()) { Engine.Index index = new Engine.Index( new Term(IdFieldMapper.NAME, Uid.encodeId(id)), - new ParsedDocument(null, null, id, null, null, sourceToParse.source(), sourceToParse.getXContentType(), null), + new ParsedDocument(null, null, id, null, null, sourceToParse.source(), sourceToParse.getMediaType(), null), seqNo, opPrimaryTerm, version, diff --git a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java index 3314b2e8d2c8c..0e9d7c2f14113 100644 --- a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java @@ -50,9 +50,9 @@ import org.opensearch.common.document.DocumentField; import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.engine.Engine; import org.opensearch.index.get.GetResult; import org.opensearch.index.mapper.DocumentMapperForType; @@ -390,13 +390,13 @@ private static ParsedDocument parseDocument( IndexShard indexShard, String index, BytesReference doc, - XContentType xContentType, + MediaType mediaType, String routing ) { MapperService mapperService = indexShard.mapperService(); DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(); ParsedDocument parsedDocument = docMapper.getDocumentMapper() - .parse(new SourceToParse(index, "_id_for_tv_api", doc, xContentType, routing)); + .parse(new SourceToParse(index, "_id_for_tv_api", doc, mediaType, routing)); if (docMapper.getMapping() != null) { parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping()); } diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java index 0984046ca3077..52ced9c051d14 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestService.java +++ b/server/src/main/java/org/opensearch/ingest/IngestService.java @@ -474,7 +474,7 @@ public static ClusterState innerPut(PutPipelineRequest request, ClusterState cur pipelines = new HashMap<>(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getMediaType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metadata( Metadata.builder(currentState.getMetadata()).putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines)).build() @@ -487,7 +487,7 @@ void validatePipeline(Map ingestInfos, PutPipelineReq throw new IllegalStateException("Ingest info is empty"); } - Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); + Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2(); Pipeline pipeline = Pipeline.create(request.getId(), pipelineConfig, processorFactories, scriptService); List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { diff --git a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java index 08a3708ae60ae..04892e4653065 100644 --- a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java @@ -32,6 +32,7 @@ package org.opensearch.ingest; +import org.opensearch.Version; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.core.ParseField; @@ -77,7 +78,7 @@ private static class Builder { private String id; private BytesReference config; - private XContentType xContentType; + private MediaType mediaType; void setId(String id) { this.id = id; @@ -88,11 +89,11 @@ void setConfig(BytesReference config, MediaType mediaType) { throw new IllegalArgumentException("PipelineConfiguration does not support media type [" + mediaType.getClass() + "]"); } this.config = config; - this.xContentType = XContentType.fromMediaType(mediaType); + this.mediaType = mediaType; } PipelineConfiguration build() { - return new PipelineConfiguration(id, config, xContentType); + return new PipelineConfiguration(id, config, mediaType); } } @@ -101,16 +102,12 @@ PipelineConfiguration build() { // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; - private final XContentType xContentType; + private final MediaType mediaType; - public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { + public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { this.id = Objects.requireNonNull(id); this.config = Objects.requireNonNull(config); - this.xContentType = Objects.requireNonNull(xContentType); - } - - public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { - this(id, config, XContentType.fromMediaType(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public String getId() { @@ -118,12 +115,12 @@ public String getId() { } public Map getConfigAsMap() { - return XContentHelper.convertToMap(config, true, xContentType).v2(); + return XContentHelper.convertToMap(config, true, mediaType).v2(); } // pkg-private for tests - XContentType getXContentType() { - return xContentType; + MediaType getMediaType() { + return mediaType; } // pkg-private for tests @@ -141,7 +138,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class)); + return new PipelineConfiguration( + in.readString(), + in.readBytesReference(), + in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readMediaType() : in.readEnum(XContentType.class) + ); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -157,7 +158,11 @@ public String toString() { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override diff --git a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java index 2fc6ef86df603..dcee6500325b9 100644 --- a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java @@ -35,9 +35,10 @@ import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import java.io.OutputStream; @@ -86,43 +87,40 @@ protected AbstractRestChannel(RestRequest request, boolean detailedErrorsEnabled @Override public XContentBuilder newBuilder() throws IOException { - return newBuilder(request.getXContentType(), true); + return newBuilder(request.getMediaType(), true); } @Override public XContentBuilder newErrorBuilder() throws IOException { // Disable filtering when building error responses - return newBuilder(request.getXContentType(), false); + return newBuilder(request.getMediaType(), false); } /** * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following - * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format - * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map + * logic. If the request has a format parameter that will be used to attempt to map to an {@link MediaType}. If there is no format + * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link MediaType}. If this first attempt to map * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls * back to JSON. */ @Override - public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException { + public XContentBuilder newBuilder(@Nullable MediaType requestContentType, boolean useFiltering) throws IOException { return newBuilder(requestContentType, null, useFiltering); } /** * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type can be sent as a parameter, - * through {@code responseContentType} or it can fallback to {@link #newBuilder(XContentType, boolean)} logic if the sent type value + * through {@code responseContentType} or it can fallback to {@link #newBuilder(MediaType, boolean)} logic if the sent type value * is {@code null}. */ @Override - public XContentBuilder newBuilder( - @Nullable XContentType requestContentType, - @Nullable XContentType responseContentType, - boolean useFiltering - ) throws IOException { + public XContentBuilder newBuilder(@Nullable MediaType requestContentType, @Nullable MediaType responseContentType, boolean useFiltering) + throws IOException { if (responseContentType == null) { // TODO should format vs acceptHeader always be the same, do we allow overriding? - responseContentType = XContentType.fromFormat(format); + responseContentType = MediaType.fromFormat(format); if (responseContentType == null) { - responseContentType = XContentType.fromMediaType(acceptHeader); + responseContentType = MediaType.fromMediaType(acceptHeader); } } // try to determine the response content type from the media type or the format query string parameter, with the format parameter @@ -134,7 +132,7 @@ public XContentBuilder newBuilder( responseContentType = requestContentType; } else { // default to JSON output when all else fails - responseContentType = XContentType.JSON; + responseContentType = MediaTypeParserRegistry.getDefaultMediaType(); } } diff --git a/server/src/main/java/org/opensearch/rest/RestChannel.java b/server/src/main/java/org/opensearch/rest/RestChannel.java index 6e9e1559ce104..b8ce3e92e0098 100644 --- a/server/src/main/java/org/opensearch/rest/RestChannel.java +++ b/server/src/main/java/org/opensearch/rest/RestChannel.java @@ -34,8 +34,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -50,9 +50,9 @@ public interface RestChannel { XContentBuilder newErrorBuilder() throws IOException; - XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException; + XContentBuilder newBuilder(@Nullable MediaType mediaType, boolean useFiltering) throws IOException; - XContentBuilder newBuilder(@Nullable XContentType xContentType, @Nullable XContentType responseContentType, boolean useFiltering) + XContentBuilder newBuilder(@Nullable MediaType mediaType, @Nullable MediaType responseContentType, boolean useFiltering) throws IOException; BytesStreamOutput bytesOutput(); diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 87d1542102f91..a2938810db1c2 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -47,6 +47,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentType; @@ -282,17 +283,17 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th private void dispatchRequest(RestRequest request, RestChannel channel, RestHandler handler) throws Exception { final int contentLength = request.content().length(); if (contentLength > 0) { - final XContentType xContentType = request.getXContentType(); - if (xContentType == null) { + final MediaType mediaType = request.getMediaType(); + if (mediaType == null) { sendContentTypeErrorMessage(request.getAllHeaderValues("Content-Type"), channel); return; } - if (handler.supportsContentStream() && xContentType != XContentType.JSON && xContentType != XContentType.SMILE) { + if (handler.supportsContentStream() && mediaType != XContentType.JSON && mediaType != XContentType.SMILE) { channel.sendResponse( BytesRestResponse.createSimpleErrorResponse( channel, RestStatus.NOT_ACCEPTABLE, - "Content-Type [" + xContentType + "] does not support stream parsing. Use JSON or SMILE instead" + "Content-Type [" + mediaType + "] does not support stream parsing. Use JSON or SMILE instead" ) ); return; @@ -592,14 +593,13 @@ public XContentBuilder newErrorBuilder() throws IOException { } @Override - public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { - return delegate.newBuilder(xContentType, useFiltering); + public XContentBuilder newBuilder(@Nullable MediaType mediaType, boolean useFiltering) throws IOException { + return delegate.newBuilder(mediaType, useFiltering); } @Override - public XContentBuilder newBuilder(XContentType xContentType, XContentType responseContentType, boolean useFiltering) - throws IOException { - return delegate.newBuilder(xContentType, responseContentType, useFiltering); + public XContentBuilder newBuilder(MediaType mediaType, MediaType responseContentType, boolean useFiltering) throws IOException { + return delegate.newBuilder(mediaType, responseContentType, useFiltering); } @Override diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 653720391d6a3..7382701b6f787 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -44,6 +44,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; @@ -84,7 +85,7 @@ public class RestRequest implements ToXContent.Params { private final Map> headers; private final String rawPath; private final Set consumedParams = new HashSet<>(); - private final SetOnce xContentType = new SetOnce<>(); + private final SetOnce mediaType = new SetOnce<>(); private final HttpChannel httpChannel; private HttpRequest httpRequest; @@ -117,14 +118,14 @@ private RestRequest( HttpChannel httpChannel, long requestId ) { - final XContentType xContentType; + final MediaType xContentType; try { xContentType = parseContentType(headers.get("Content-Type")); } catch (final IllegalArgumentException e) { throw new ContentTypeHeaderException(e); } if (xContentType != null) { - this.xContentType.set(xContentType); + this.mediaType.set(xContentType); } this.xContentRegistry = xContentRegistry; this.httpRequest = httpRequest; @@ -295,7 +296,7 @@ protected BytesReference content(final boolean contentConsumed) { public final BytesReference requiredContent() { if (hasContent() == false) { throw new OpenSearchParseException("request body is required"); - } else if (xContentType.get() == null) { + } else if (mediaType.get() == null) { throw new IllegalStateException("unknown content type"); } return content(); @@ -340,8 +341,8 @@ public final long getRequestId() { * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request */ @Nullable - public final XContentType getXContentType() { - return xContentType.get(); + public final MediaType getMediaType() { + return mediaType.get(); } public HttpChannel getHttpChannel() { @@ -486,7 +487,7 @@ public NamedXContentRegistry getXContentRegistry() { */ public final XContentParser contentParser() throws IOException { BytesReference content = requiredContent(); // will throw exception if body or content type missing - return xContentType.get().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); + return mediaType.get().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); } /** @@ -514,7 +515,7 @@ public final boolean hasContentOrSourceParam() { * if you need to handle the absence request content gracefully. */ public final XContentParser contentOrSourceParamParser() throws IOException { - Tuple tuple = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); return tuple.v1().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, tuple.v2().streamInput()); } @@ -525,12 +526,12 @@ public final XContentParser contentOrSourceParamParser() throws IOException { */ public final void withContentOrSourceParamParserOrNull(CheckedConsumer withParser) throws IOException { if (hasContentOrSourceParam()) { - Tuple tuple = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); BytesReference content = tuple.v2(); - XContentType xContentType = tuple.v1(); + MediaType mediaType = tuple.v1(); try ( InputStream stream = content.streamInput(); - XContentParser parser = xContentType.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream) + XContentParser parser = mediaType.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream) ) { withParser.accept(parser); } @@ -543,11 +544,11 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() { + public final Tuple contentOrSourceParam() { if (hasContentOrSourceParam() == false) { throw new OpenSearchParseException("request body or source parameter is required"); } else if (hasContent()) { - return new Tuple<>(xContentType.get(), requiredContent()); + return new Tuple<>(mediaType.get(), requiredContent()); } String source = param("source"); String typeParam = param("source_content_type"); @@ -555,18 +556,18 @@ public final Tuple contentOrSourceParam() { throw new IllegalStateException("source and source_content_type parameters are required"); } BytesArray bytes = new BytesArray(source); - final XContentType xContentType = parseContentType(Collections.singletonList(typeParam)); - if (xContentType == null) { + final MediaType mediaType = parseContentType(Collections.singletonList(typeParam)); + if (mediaType == null) { throw new IllegalStateException("Unknown value for source_content_type [" + typeParam + "]"); } - return new Tuple<>(xContentType, bytes); + return new Tuple<>(mediaType, bytes); } /** * Parses the given content type string for the media type. This method currently ignores parameters. */ // TODO stop ignoring parameters such as charset... - public static XContentType parseContentType(List header) { + public static MediaType parseContentType(List header) { if (header == null || header.isEmpty()) { return null; } else if (header.size() > 1) { @@ -580,7 +581,7 @@ public static XContentType parseContentType(List header) { if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { - return XContentType.fromMediaType(elements[0]); + return MediaType.fromMediaType(elements[0]); } else { throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java index 28381af769a51..47ab7ae585039 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java @@ -35,7 +35,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.logging.DeprecationLogger; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -80,10 +80,10 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client String id = request.param("id"); String context = request.param("context"); BytesReference content = request.requiredContent(); - XContentType xContentType = request.getXContentType(); - StoredScriptSource source = StoredScriptSource.parse(content, xContentType); + MediaType mediaType = request.getMediaType(); + StoredScriptSource source = StoredScriptSource.parse(content, mediaType); - PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getXContentType(), source); + PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getMediaType(), source); putRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", putRequest.clusterManagerNodeTimeout())); parseDeprecatedMasterTimeoutParameter(putRequest, request, deprecationLogger, getName()); putRequest.timeout(request.paramAsTime("timeout", putRequest.timeout())); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java index e7335eee89c5a..d987e8fe63506 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -76,7 +76,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2(); sourceAsMap = prepareMappings(sourceAsMap); createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 86b0111e4b064..dde9f470161b9 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -88,7 +88,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2(); sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); putRequest.source(sourceAsMap); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java index fd293dceb4cc1..8fdf000139d89 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java @@ -83,7 +83,7 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2(); if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 556928d4685a0..6a5d4e40eb452 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -42,8 +42,8 @@ import org.opensearch.common.unit.SizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; @@ -69,18 +69,18 @@ public class RestTable { public static RestResponse buildResponse(Table table, RestChannel channel) throws Exception { RestRequest request = channel.request(); - XContentType xContentType = getXContentType(request); + MediaType xContentType = getXContentType(request); if (xContentType != null) { return buildXContentBuilder(table, channel); } return buildTextPlainResponse(table, channel); } - private static XContentType getXContentType(RestRequest request) { + private static MediaType getXContentType(RestRequest request) { if (request.hasParam("format")) { - return XContentType.fromFormat(request.param("format")); + return MediaType.fromFormat(request.param("format")); } - return XContentType.fromMediaType(request.header("Accept")); + return MediaType.fromMediaType(request.header("Accept")); } public static RestResponse buildXContentBuilder(Table table, RestChannel channel) throws Exception { diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java index 3fc02db0a8365..b046146707885 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java @@ -105,7 +105,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC defaultPipeline, defaultRequireAlias, allowExplicitIndex, - request.getXContentType() + request.getMediaType() ); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java index 7247067755721..b4abe449e027f 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java @@ -108,7 +108,7 @@ static class RestGetSourceResponseListener extends RestResponseListener sourceTuple = restRequest.contentOrSourceParam(); + Tuple sourceTuple = restRequest.contentOrSourceParam(); PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.clusterManagerNodeTimeout(restRequest.paramAsTime("cluster_manager_timeout", request.clusterManagerNodeTimeout())); parseDeprecatedMasterTimeoutParameter(request, restRequest, deprecationLogger, getName()); diff --git a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java index ec8cc2c00e65a..1aa2c11e3f05e 100644 --- a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java @@ -36,7 +36,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -75,7 +75,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - Tuple sourceTuple = restRequest.contentOrSourceParam(); + Tuple sourceTuple = restRequest.contentOrSourceParam(); SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1()); request.setId(restRequest.param("id")); request.setVerbose(restRequest.paramAsBoolean("verbose", false)); diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java index 189b1e93e6444..5be5a54edddf5 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java @@ -45,9 +45,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestCancellableNodeClient; @@ -190,7 +190,7 @@ public static void parseMultiLineRequest( boolean ccsMinimizeRoundtrips = request.paramAsBoolean("ccs_minimize_roundtrips", true); String routing = request.param("routing"); - final Tuple sourceTuple = request.contentOrSourceParam(); + final Tuple sourceTuple = request.contentOrSourceParam(); final XContent xContent = sourceTuple.v1().xContent(); final BytesReference data = sourceTuple.v2(); MultiSearchRequest.readMultiLineFormat( diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java index 2f398a6bea9ff..73eead1b91d57 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java @@ -12,7 +12,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -40,7 +40,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - Tuple sourceTuple = restRequest.contentOrSourceParam(); + Tuple sourceTuple = restRequest.contentOrSourceParam(); PutSearchPipelineRequest request = new PutSearchPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.clusterManagerNodeTimeout(restRequest.paramAsTime("cluster_manager_timeout", request.clusterManagerNodeTimeout())); request.timeout(restRequest.paramAsTime("timeout", request.timeout())); diff --git a/server/src/main/java/org/opensearch/script/StoredScriptSource.java b/server/src/main/java/org/opensearch/script/StoredScriptSource.java index 1d6f2b8e4ccea..9d3af9e4c9caf 100644 --- a/server/src/main/java/org/opensearch/script/StoredScriptSource.java +++ b/server/src/main/java/org/opensearch/script/StoredScriptSource.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; @@ -197,71 +198,72 @@ private StoredScriptSource build(boolean ignoreEmpty) { /** * This will parse XContent into a {@link StoredScriptSource}. The following formats can be parsed: - * + *

    * The simple script format with no compiler options or user-defined params: - * + *

    * Example: * {@code * {"script": "return Math.log(doc.popularity) * 100;"} * } - * + *

    * The above format requires the lang to be specified using the deprecated stored script namespace * (as a url parameter during a put request). See {@link ScriptMetadata} for more information about * the stored script namespaces. - * + *

    * The complex script format using the new stored script namespace * where lang and source are required but options is optional: - * + *

    * {@code * { - * "script" : { - * "lang" : "", - * "source" : "", - * "options" : { - * "option0" : "", - * "option1" : "", - * ... - * } - * } + * "script" : { + * "lang" : "", + * "source" : "", + * "options" : { + * "option0" : "", + * "option1" : "", + * ... * } * } - * + * } + * } + *

    * Example: * {@code * { - * "script": { - * "lang" : "painless", - * "source" : "return Math.log(doc.popularity) * params.multiplier" - * } + * "script": { + * "lang" : "painless", + * "source" : "return Math.log(doc.popularity) * params.multiplier" * } * } - * + * } + *

    * The use of "source" may also be substituted with "code" for backcompat with 5.3 to 5.5 format. For example: - * + *

    * {@code * { - * "script" : { - * "lang" : "", - * "code" : "", - * "options" : { - * "option0" : "", - * "option1" : "", - * ... - * } - * } + * "script" : { + * "lang" : "", + * "code" : "", + * "options" : { + * "option0" : "", + * "option1" : "", + * ... * } * } - * + * } + * } + *

    * Note that the "source" parameter can also handle template parsing including from * a complex JSON object. * - * @param content The content from the request to be parsed as described above. - * @return The parsed {@link StoredScriptSource}. + * @param content The content from the request to be parsed as described above. + * @param mediaType The media type of the request + * @return The parsed {@link StoredScriptSource}. */ - public static StoredScriptSource parse(BytesReference content, XContentType xContentType) { + public static StoredScriptSource parse(BytesReference content, MediaType mediaType) { try ( InputStream stream = content.streamInput(); - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream) ) { Token token = parser.nextToken(); diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java index b82764c38f747..97a6093ea39d1 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java @@ -52,6 +52,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.IndexSettings; import org.opensearch.index.fieldvisitor.CustomFieldsVisitor; import org.opensearch.index.fieldvisitor.FieldsVisitor; @@ -377,7 +378,7 @@ private HitContext prepareNestedHitContext( String rootId; Map rootSourceAsMap = null; - XContentType rootSourceContentType = null; + MediaType rootSourceContentType = null; int nestedDocId = nestedTopDocId - subReaderContext.docBase; diff --git a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java index 1d4a424550c12..1341fc0fdabb3 100644 --- a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java @@ -75,8 +75,8 @@ public Map source() { return source; } - public XContentType sourceContentType() { - return XContentType.fromMediaType(sourceContentType); + public MediaType sourceContentType() { + return sourceContentType; } public int docId() { @@ -155,7 +155,7 @@ public void setSource(BytesReference source) { this.sourceAsBytes = source; } - public void setSourceContentType(XContentType sourceContentType) { + public void setSourceContentType(MediaType sourceContentType) { this.sourceContentType = sourceContentType; } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java index 512e400c54784..b4f6549c83390 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java @@ -8,6 +8,7 @@ package org.opensearch.search.pipeline; +import org.opensearch.Version; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.common.Strings; @@ -56,7 +57,7 @@ private static class Builder { private String id; private BytesReference config; - private XContentType xContentType; + private MediaType mediaType; void setId(String id) { this.id = id; @@ -67,11 +68,11 @@ void setConfig(BytesReference config, MediaType mediaType) { throw new IllegalArgumentException("PipelineConfiguration does not support media type [" + mediaType.getClass() + "]"); } this.config = config; - this.xContentType = XContentType.fromMediaType(mediaType); + this.mediaType = mediaType; } PipelineConfiguration build() { - return new PipelineConfiguration(id, config, xContentType); + return new PipelineConfiguration(id, config, mediaType); } } @@ -80,16 +81,12 @@ PipelineConfiguration build() { // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; - private final XContentType xContentType; + private final MediaType mediaType; - public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { + public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { this.id = Objects.requireNonNull(id); this.config = Objects.requireNonNull(config); - this.xContentType = Objects.requireNonNull(xContentType); - } - - public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { - this(id, config, XContentType.fromMediaType(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public String getId() { @@ -97,12 +94,12 @@ public String getId() { } public Map getConfigAsMap() { - return XContentHelper.convertToMap(config, true, xContentType).v2(); + return XContentHelper.convertToMap(config, true, mediaType).v2(); } // pkg-private for tests - XContentType getXContentType() { - return xContentType; + MediaType getMediaType() { + return mediaType; } // pkg-private for tests @@ -120,7 +117,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class)); + return new PipelineConfiguration( + in.readString(), + in.readBytesReference(), + in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readMediaType() : in.readEnum(XContentType.class) + ); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -136,7 +137,11 @@ public String toString() { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 7384b494d71fc..1066d836e5183 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -259,7 +259,7 @@ static ClusterState innerPut(PutSearchPipelineRequest request, ClusterState curr } else { pipelines = new HashMap<>(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getMediaType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metadata( Metadata.builder(currentState.getMetadata()) @@ -273,7 +273,7 @@ void validatePipeline(Map searchPipelineInfos if (searchPipelineInfos.isEmpty()) { throw new IllegalStateException("Search pipeline info is empty"); } - Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); + Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2(); Pipeline pipeline = PipelineWithMetrics.create( request.getId(), pipelineConfig, diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index c2163ae1d490a..b8d7d130e846b 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -59,8 +59,10 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.core.xcontent.MediaTypeParserRegistry; import org.opensearch.node.NodeClosedException; import org.opensearch.node.ReportingService; import org.opensearch.tasks.Task; @@ -169,7 +171,10 @@ public void close() {} * over the {@link StreamOutput} and {@link StreamInput} wire */ Streamables.registerStreamables(); + /** Registers OpenSearch server specific exceptions (exceptions outside of core library) */ OpenSearchServerException.registerExceptions(); + // set the default media type to JSON (fallback if a media type is not specified) + MediaTypeParserRegistry.setDefaultMediaType(XContentType.JSON); } /** does nothing. easy way to ensure class is loaded so the above static block is called to register the streamables */ diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index 6ba7961ccecf3..e92096f139d17 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -244,7 +244,7 @@ private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOExcep FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry()); requestBuilder.withParams(params); if (hasBody) { - requestBuilder.withContent(BytesReference.bytes(builder), XContentType.fromMediaType(builder.contentType())); + requestBuilder.withContent(BytesReference.bytes(builder), builder.contentType()); } return requestBuilder.build(); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java index 2eb6fa504baf2..cfdd776e60832 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -56,13 +56,13 @@ public void testSerialization() throws IOException { new StoredScriptSource("foo", "bar", Collections.emptyMap()) ); - assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); + assertEquals(XContentType.JSON, storedScriptRequest.mediaType()); try (BytesStreamOutput output = new BytesStreamOutput()) { storedScriptRequest.writeTo(output); try (StreamInput in = output.bytes().streamInput()) { PutStoredScriptRequest serialized = new PutStoredScriptRequest(in); - assertEquals(XContentType.JSON, serialized.xContentType()); + assertEquals(XContentType.JSON, serialized.mediaType()); assertEquals(storedScriptRequest.id(), serialized.id()); assertEquals(storedScriptRequest.context(), serialized.context()); } diff --git a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java index a86257fd741c0..336ec67546dc5 100644 --- a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java @@ -49,14 +49,14 @@ public class PutPipelineRequestTests extends OpenSearchTestCase { public void testSerializationWithXContent() throws IOException { PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); - assertEquals(XContentType.JSON, request.getXContentType()); + assertEquals(XContentType.JSON, request.getMediaType()); BytesStreamOutput output = new BytesStreamOutput(); request.writeTo(output); StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); PutPipelineRequest serialized = new PutPipelineRequest(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals(XContentType.JSON, serialized.getMediaType()); assertEquals("{}", serialized.getSource().utf8ToString()); } diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java index 53607f432bbd5..8c53d7edebca8 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java @@ -44,81 +44,80 @@ public class XContentTypeTests extends OpenSearchTestCase { public void testFromJson() throws Exception { String mediaType = "application/json"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromNdJson() throws Exception { String mediaType = "application/x-ndjson"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromJsonUppercase() throws Exception { String mediaType = "application/json".toUpperCase(Locale.ROOT); XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromYaml() throws Exception { String mediaType = "application/yaml"; XContentType expectedXContentType = XContentType.YAML; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromSmile() throws Exception { String mediaType = "application/smile"; XContentType expectedXContentType = XContentType.SMILE; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromCbor() throws Exception { String mediaType = "application/cbor"; XContentType expectedXContentType = XContentType.CBOR; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromWildcard() throws Exception { String mediaType = "application/*"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromWildcardUppercase() throws Exception { String mediaType = "APPLICATION/*"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromRubbish() throws Exception { - assertThat(XContentType.fromMediaType((String) null), nullValue()); - assertThat(XContentType.fromMediaType((MediaType) null), nullValue()); - assertThat(XContentType.fromMediaType(""), nullValue()); - assertThat(XContentType.fromMediaType("text/plain"), nullValue()); - assertThat(XContentType.fromMediaType("gobbly;goop"), nullValue()); + assertThat(MediaType.fromMediaType(null), nullValue()); + assertThat(MediaType.fromMediaType(""), nullValue()); + assertThat(MediaType.fromMediaType("text/plain"), nullValue()); + assertThat(MediaType.fromMediaType("gobbly;goop"), nullValue()); } public void testVersionedMediaType() throws Exception { - assertThat(XContentType.fromMediaType("application/vnd.opensearch+json;compatible-with=7"), equalTo(XContentType.JSON)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+yaml;compatible-with=7"), equalTo(XContentType.YAML)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+cbor;compatible-with=7"), equalTo(XContentType.CBOR)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+smile;compatible-with=7"), equalTo(XContentType.SMILE)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+json;compatible-with=7"), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+yaml;compatible-with=7"), equalTo(XContentType.YAML)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+cbor;compatible-with=7"), equalTo(XContentType.CBOR)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+smile;compatible-with=7"), equalTo(XContentType.SMILE)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+json ;compatible-with=7"), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+json ;compatible-with=7"), equalTo(XContentType.JSON)); String mthv = "application/vnd.opensearch+json ;compatible-with=7;charset=utf-8"; - assertThat(XContentType.fromMediaType(mthv), equalTo(XContentType.JSON)); - assertThat(XContentType.fromMediaType(mthv.toUpperCase(Locale.ROOT)), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType(mthv), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType(mthv.toUpperCase(Locale.ROOT)), equalTo(XContentType.JSON)); } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java index 365d3738f4022..0e04e57a290b6 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java @@ -367,9 +367,7 @@ private void doTestDefaultFloatingPointMappings(DocumentMapper mapper, XContentB .field("quux", "3.2") // float detected through numeric detection .endObject() ); - ParsedDocument parsedDocument = mapper.parse( - new SourceToParse("index", "id", source, XContentType.fromMediaType(builder.contentType())) - ); + ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "id", source, builder.contentType())); Mapping update = parsedDocument.dynamicMappingsUpdate(); assertNotNull(update); assertThat(((FieldMapper) update.root().getMapper("foo")).fieldType().typeName(), equalTo("float")); diff --git a/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java index 51ebd1b1f7ad5..aed08c2d1875a 100644 --- a/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java @@ -58,13 +58,13 @@ public void testSerialization() throws IOException { new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON ); - assertEquals(XContentType.JSON, configuration.getXContentType()); + assertEquals(XContentType.JSON, configuration.getMediaType()); BytesStreamOutput out = new BytesStreamOutput(); configuration.writeTo(out); StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); PipelineConfiguration serialized = PipelineConfiguration.readFrom(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals(XContentType.JSON, serialized.getMediaType()); assertEquals("{}", serialized.getConfig().utf8ToString()); } @@ -83,8 +83,8 @@ public void testParser() throws IOException { XContentParser xContentParser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput()); PipelineConfiguration parsed = parser.parse(xContentParser, null); - assertEquals(xContentType, parsed.getXContentType()); - assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); + assertEquals(xContentType, parsed.getMediaType()); + assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getMediaType())); assertEquals("1", parsed.getId()); } diff --git a/server/src/test/java/org/opensearch/rest/RestRequestTests.java b/server/src/test/java/org/opensearch/rest/RestRequestTests.java index 15066362555ef..97350824dd1e4 100644 --- a/server/src/test/java/org/opensearch/rest/RestRequestTests.java +++ b/server/src/test/java/org/opensearch/rest/RestRequestTests.java @@ -203,12 +203,12 @@ public void testContentTypeParsing() { Map> map = new HashMap<>(); map.put("Content-Type", Collections.singletonList(xContentType.mediaType())); RestRequest restRequest = contentRestRequest("", Collections.emptyMap(), map); - assertEquals(xContentType, restRequest.getXContentType()); + assertEquals(xContentType, restRequest.getMediaType()); map = new HashMap<>(); map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters())); restRequest = contentRestRequest("", Collections.emptyMap(), map); - assertEquals(xContentType, restRequest.getXContentType()); + assertEquals(xContentType, restRequest.getMediaType()); } } @@ -221,7 +221,7 @@ public void testPlainTextSupport() { Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8")) ) ); - assertNull(restRequest.getXContentType()); + assertNull(restRequest.getMediaType()); } public void testMalformedContentTypeHeader() { @@ -237,7 +237,7 @@ public void testMalformedContentTypeHeader() { public void testNoContentTypeHeader() { RestRequest contentRestRequest = contentRestRequest("", Collections.emptyMap(), Collections.emptyMap()); - assertNull(contentRestRequest.getXContentType()); + assertNull(contentRestRequest.getMediaType()); } public void testMultipleContentTypeHeaders() { diff --git a/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java b/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java index bcab5dc7b776a..83e7a3712a9ad 100644 --- a/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java @@ -36,6 +36,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -130,13 +131,13 @@ public void testGetScript() throws Exception { .endObject() .endObject() .endObject(); - XContentType xContentType = XContentType.fromMediaType(sourceBuilder.contentType()); - builder.storeScript("source_template", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), xContentType)); + MediaType mediaType = sourceBuilder.contentType(); + builder.storeScript("source_template", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), mediaType)); sourceBuilder = XContentFactory.jsonBuilder(); - xContentType = XContentType.fromMediaType(sourceBuilder.contentType()); + mediaType = sourceBuilder.contentType(); sourceBuilder.startObject().startObject("script").field("lang", "_lang").field("source", "_source").endObject().endObject(); - builder.storeScript("script", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), xContentType)); + builder.storeScript("script", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), mediaType)); ScriptMetadata scriptMetadata = builder.build(); assertEquals("_source", scriptMetadata.getStoredScript("script").getSource()); @@ -303,7 +304,7 @@ private ScriptMetadata randomScriptMetadata(XContentType sourceContentType, int .endObject(); builder.storeScript( randomAlphaOfLength(i + 1), - StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), XContentType.fromMediaType(sourceBuilder.contentType())) + StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), sourceBuilder.contentType()) ); } return builder.build(); diff --git a/test/framework/src/main/java/org/opensearch/test/RandomObjects.java b/test/framework/src/main/java/org/opensearch/test/RandomObjects.java index e81f0df9a5ac4..a328e6778dfaf 100644 --- a/test/framework/src/main/java/org/opensearch/test/RandomObjects.java +++ b/test/framework/src/main/java/org/opensearch/test/RandomObjects.java @@ -98,7 +98,7 @@ public static Tuple, List> randomStoredFieldValues(Random r List originalValues = randomStoredFieldValues(random, numValues); List expectedParsedValues = new ArrayList<>(numValues); for (Object originalValue : originalValues) { - expectedParsedValues.add(getExpectedParsedValue(XContentType.fromMediaType(mediaType), originalValue)); + expectedParsedValues.add(getExpectedParsedValue(mediaType, originalValue)); } return Tuple.tuple(originalValues, expectedParsedValues); } @@ -154,15 +154,15 @@ private static List randomStoredFieldValues(Random random, int numValues * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via * {@link org.opensearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans. */ - public static Object getExpectedParsedValue(XContentType xContentType, Object value) { + public static Object getExpectedParsedValue(MediaType mediaType, Object value) { if (value instanceof BytesArray) { - if (xContentType == XContentType.JSON) { + if (mediaType == XContentType.JSON) { // JSON writes base64 format return Base64.getEncoder().encodeToString(((BytesArray) value).toBytesRef().bytes); } } if (value instanceof Float) { - if (xContentType == XContentType.CBOR || xContentType == XContentType.SMILE) { + if (mediaType == XContentType.CBOR || mediaType == XContentType.SMILE) { // with binary content types we pass back the object as is return value; } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java index 375768ed12cc4..2bbaf8c80dfa4 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java @@ -35,8 +35,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; import org.opensearch.http.HttpResponse; @@ -231,10 +231,10 @@ public Builder withParams(Map params) { return this; } - public Builder withContent(BytesReference content, XContentType xContentType) { + public Builder withContent(BytesReference content, MediaType mediaType) { this.content = content; - if (xContentType != null) { - headers.put("Content-Type", Collections.singletonList(xContentType.mediaType())); + if (mediaType != null) { + headers.put("Content-Type", Collections.singletonList(mediaType.mediaType())); } return this; } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java index 512369fdfe8a1..6d3c2eaa972e6 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java @@ -66,6 +66,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; @@ -143,10 +144,10 @@ public abstract class OpenSearchRestTestCase extends OpenSearchTestCase { * Convert the entity from a {@link Response} into a map of maps. */ public static Map entityAsMap(Response response) throws IOException { - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType()); + MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType()); // EMPTY and THROW are fine here because `.map` doesn't use named x content or deprecation try ( - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, @@ -161,10 +162,10 @@ public static Map entityAsMap(Response response) throws IOExcept * Convert the entity from a {@link Response} into a list of maps. */ public static List entityAsList(Response response) throws IOException { - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType()); + MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType()); // EMPTY and THROW are fine here because `.map` doesn't use named x content or deprecation try ( - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, @@ -1074,7 +1075,7 @@ protected static Map getAsMap(final String endpoint) throws IOEx } protected static Map responseAsMap(Response response) throws IOException { - XContentType entityContentType = XContentType.fromMediaType(response.getEntity().getContentType()); + MediaType entityContentType = MediaType.fromMediaType(response.getEntity().getContentType()); Map responseEntity = XContentHelper.convertToMap( entityContentType.xContent(), response.getEntity().getContent(), diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java index cb460f17cbec0..635dc49ff5166 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -41,6 +41,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.client.NodeSelector; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -138,20 +139,20 @@ private HttpEntity createEntity(List> bodies, Map bytesRefList = new ArrayList<>(bodies.size()); int totalBytesLength = 0; for (Map body : bodies) { - BytesRef bytesRef = bodyAsBytesRef(body, xContentType); + BytesRef bytesRef = bodyAsBytesRef(body, mediaType); bytesRefList.add(bytesRef); totalBytesLength += bytesRef.length - bytesRef.offset + 1; } @@ -161,20 +162,20 @@ private HttpEntity createEntity(List> bodies, Map headers, XContentType[] supportedContentTypes) { - XContentType xContentType = null; + private MediaType getContentType(Map headers, XContentType[] supportedContentTypes) { + MediaType mediaType = null; String contentType = headers.get("Content-Type"); if (contentType != null) { - xContentType = XContentType.fromMediaType(contentType); + mediaType = MediaType.fromMediaType(contentType); } - if (xContentType != null) { - return xContentType; + if (mediaType != null) { + return mediaType; } if (randomizeContentType) { return RandomizedTest.randomFrom(supportedContentTypes); @@ -182,9 +183,9 @@ private XContentType getContentType(Map headers, XContentType[] return XContentType.JSON; } - private BytesRef bodyAsBytesRef(Map bodyAsMap, XContentType xContentType) throws IOException { + private BytesRef bodyAsBytesRef(Map bodyAsMap, MediaType mediaType) throws IOException { Map finalBodyAsMap = stash.replaceStashedValues(bodyAsMap); - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = XContentFactory.contentBuilder(mediaType)) { return BytesReference.bytes(builder.map(finalBodyAsMap)).toBytesRef(); } } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java index 5c6190ab860b2..0f14461eb0f86 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java @@ -38,6 +38,7 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -58,7 +59,7 @@ public class ClientYamlTestResponse { private final Response response; private final byte[] body; - private final XContentType bodyContentType; + private final MediaType bodyContentType; private ObjectPath parsedResponse; private String bodyAsString; @@ -66,7 +67,7 @@ public ClientYamlTestResponse(Response response) throws IOException { this.response = response; if (response.getEntity() != null) { String contentType = response.getHeader("Content-Type"); - this.bodyContentType = XContentType.fromMediaType(contentType); + this.bodyContentType = MediaType.fromMediaType(contentType); try { byte[] bytes = EntityUtils.toByteArray(response.getEntity()); // skip parsing if we got text back (e.g. if we called _cat apis) diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java index 8b62f0d0a9783..6c1510f8ca60a 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java @@ -36,11 +36,11 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; @@ -57,8 +57,8 @@ public class ObjectPath { public static ObjectPath createFromResponse(Response response) throws IOException { byte[] bytes = EntityUtils.toByteArray(response.getEntity()); String contentType = response.getHeader("Content-Type"); - XContentType xContentType = XContentType.fromMediaType(contentType); - return ObjectPath.createFromXContent(xContentType.xContent(), new BytesArray(bytes)); + MediaType mediaType = MediaType.fromMediaType(contentType); + return ObjectPath.createFromXContent(mediaType.xContent(), new BytesArray(bytes)); } public static ObjectPath createFromXContent(XContent xContent, BytesReference input) throws IOException { From 5d78de61f33bd813913563076d3dfcf8ddaf1498 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Mon, 24 Jul 2023 15:52:32 -0500 Subject: [PATCH 155/186] Retry only specified tests (#8825) This list was compiled by finding every test failure that succeeded on a retry in the past month (approx. June 20-July 20, 2023). The [flaky-test-finder.rb][1] script was used to collect this list. ``` ruby flaky-test-finder.rb -s 18000 -e 20681 | sed 's/[0-9]*[[:space:]]\(.*\)\..*/includeClasses\.add(\"\1\")/' | sort | uniq ``` [1]: https://gist.github.com/andrross/ee07a8a05beb63f1173bcb98523918b9 Signed-off-by: Andrew Ross --- TESTING.md | 5 ++++ build.gradle | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/TESTING.md b/TESTING.md index 93b4615da6f0b..a76ee07e1faac 100644 --- a/TESTING.md +++ b/TESTING.md @@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Test groups](#test-groups) - [Load balancing and caches](#load-balancing-and-caches) - [Test compatibility](#test-compatibility) + - [Retries](#retries) - [Miscellaneous](#miscellaneous) - [Running verification tasks](#running-verification-tasks) - [Testing the REST layer](#testing-the-rest-layer) @@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to ./gradlew test -Dtests.compatibility=1.0.0 +## Retries + +The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label. + ## Miscellaneous Run all tests without stopping on errors (inspect log files). diff --git a/build.gradle b/build.gradle index 88c485dae9350..c7b6987b1103f 100644 --- a/build.gradle +++ b/build.gradle @@ -470,6 +470,82 @@ subprojects { maxFailures = 10 } failOnPassedAfterRetry = false + classRetry { + includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests") + includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT") + includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT") + includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT") + includeClasses.add("org.opensearch.aliases.IndexAliasesIT") + includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.blocks.SimpleBlocksIT") + includeClasses.add("org.opensearch.client.PitIT") + includeClasses.add("org.opensearch.client.ReindexIT") + includeClasses.add("org.opensearch.cluster.ClusterHealthIT") + includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT") + includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT") + includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT") + includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests") + includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests") + includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT") + includeClasses.add("org.opensearch.cluster.service.MasterServiceTests") + includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests") + includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT") + includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT") + includeClasses.add("org.opensearch.http.SearchRestCancellationIT") + includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests") + includeClasses.add("org.opensearch.index.IndexServiceTests") + includeClasses.add("org.opensearch.index.IndexSettingsTests") + includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT") + includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests") + includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests") + includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests") + includeClasses.add("org.opensearch.index.shard.IndexShardIT") + includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests") + includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests") + includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT") + includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests") + includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT") + includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests") + includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT") + includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests") + includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT") + includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT") + includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT") + includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT") + includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT") + includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT") + includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT") + includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT") + includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT") + includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests") + } } } } From 077e12c0cf32e42500949c9e39c6fba83feca76d Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Mon, 24 Jul 2023 14:50:28 -0700 Subject: [PATCH 156/186] Fix versions for 2.x on main (#8846) Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3552c66c83229..4bc645ed1bd6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,4 +103,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x diff --git a/README.md b/README.md index 55ff067e4d779..b5fc45509b002 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.8 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.8.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.8.0") +[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0") [![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) From a3baa68b7b014520d257efbb0d4b13f66e134d12 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 24 Jul 2023 16:44:54 -0700 Subject: [PATCH 157/186] Fix flaky test SegmentReplicationRemoteStoreIT.testPressureServiceStats (#8827) * Fix ReplicationTracker to not include unavailable former primary shards when computing replication stats. Signed-off-by: Marc Handalian * Fix relocation IT relying on stats to determine if segrep has occured. The API should still show a result for the replica even if it has not sync'd. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../replication/SegmentReplicationIT.java | 81 +++++++++++-------- .../SegmentReplicationRelocationIT.java | 22 ++++- .../index/seqno/ReplicationTracker.java | 72 +++++++++++------ .../checkpoint/ReplicationCheckpoint.java | 4 + .../index/seqno/ReplicationTrackerTests.java | 54 +++++++------ 5 files changed, 150 insertions(+), 83 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index d345578c7de6e..2a7e8e58b2d03 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -875,69 +875,84 @@ public void testPressureServiceStats() throws Exception { waitForDocs(initialDocCount, indexer); refresh(INDEX_NAME); + // get shard references. + final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME); + final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); + logger.info("Replica aid {}", replicaShard.routingEntry().allocationId()); + logger.info("former primary aid {}", primaryShard.routingEntry().allocationId()); + + // fetch pressure stats from the Primary's Node. SegmentReplicationPressureService pressureService = internalCluster().getInstance( SegmentReplicationPressureService.class, primaryNode ); - final Map shardStats = pressureService.nodeStats().getShardStats(); - assertEquals(1, shardStats.size()); - final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME); - IndexShard replica = getIndexShard(replicaNode, INDEX_NAME); - SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId()); - Set replicaStats = groupStats.getReplicaStats(); - assertEquals(1, replicaStats.size()); - - // assert replica node returns nothing. + // Fetch pressure stats from the Replica's Node we will assert replica node returns nothing until it is promoted. SegmentReplicationPressureService replicaNode_service = internalCluster().getInstance( SegmentReplicationPressureService.class, replicaNode ); + + final Map shardStats = pressureService.nodeStats().getShardStats(); + assertEquals("We should have stats returned for the replication group", 1, shardStats.size()); + + SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId()); + Set replicaStats = groupStats.getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(replicaShard.routingEntry().allocationId().getId()), replicaStats); + assertTrue(replicaNode_service.nodeStats().getShardStats().isEmpty()); - // drop the primary, this won't hand off SR state. + // drop the primary, this won't hand off pressure stats between old/new primary. internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); ensureYellowAndNoInitializingShards(INDEX_NAME); - replicaNode_service = internalCluster().getInstance(SegmentReplicationPressureService.class, replicaNode); - replica = getIndexShard(replicaNode, INDEX_NAME); - assertTrue("replica should be promoted as a primary", replica.routingEntry().primary()); - assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); - // we don't have a replica assigned yet, so this should be 0. - assertEquals(0, replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats().size()); + + assertTrue("replica should be promoted as a primary", replicaShard.routingEntry().primary()); + assertEquals( + "We should have stats returned for the replication group", + 1, + replicaNode_service.nodeStats().getShardStats().size() + ); + // after the primary is dropped and replica is promoted we won't have a replica assigned yet, so stats per replica should return + // empty. + replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats(); + assertTrue(replicaStats.isEmpty()); // start another replica. String replicaNode_2 = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); - String docId = String.valueOf(initialDocCount + 1); - client().prepareIndex(INDEX_NAME).setId(docId).setSource("foo", "bar").get(); - refresh(INDEX_NAME); - waitForSearchableDocs(initialDocCount + 1, replicaNode_2); + final IndexShard secondReplicaShard = getIndexShard(replicaNode_2, INDEX_NAME); + final String second_replica_aid = secondReplicaShard.routingEntry().allocationId().getId(); + waitForSearchableDocs(initialDocCount, replicaNode_2); - replicaNode_service = internalCluster().getInstance(SegmentReplicationPressureService.class, replicaNode); - replica = getIndexShard(replicaNode_2, INDEX_NAME); - assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); - replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats(); - assertEquals(1, replicaStats.size()); + assertEquals( + "We should have stats returned for the replication group", + 1, + replicaNode_service.nodeStats().getShardStats().size() + ); + replicaStats = replicaNode_service.nodeStats().getShardStats().get(replicaShard.shardId()).getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), replicaStats); + final SegmentReplicationShardStats replica_entry = replicaStats.stream().findFirst().get(); + assertEquals(replica_entry.getCheckpointsBehindCount(), 0); // test a checkpoint without any new segments flush(INDEX_NAME); assertBusy(() -> { - final SegmentReplicationPressureService service = internalCluster().getInstance( - SegmentReplicationPressureService.class, - replicaNode - ); - assertEquals(1, service.nodeStats().getShardStats().size()); - final Set shardStatsSet = service.nodeStats() + assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); + final Set shardStatsSet = replicaNode_service.nodeStats() .getShardStats() - .get(primaryShard.shardId()) + .get(replicaShard.shardId()) .getReplicaStats(); - assertEquals(1, shardStatsSet.size()); + assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), shardStatsSet); final SegmentReplicationShardStats stats = shardStatsSet.stream().findFirst().get(); assertEquals(0, stats.getCheckpointsBehindCount()); }); } } + private void assertAllocationIdsInReplicaShardStats(Set expected, Set replicaStats) { + assertEquals(expected, replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet())); + } + /** * Tests a scroll query on the replica * @throws Exception when issue is encountered diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java index 7426ca2f13f84..7cf7e5148dd4a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java @@ -22,6 +22,8 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -29,8 +31,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -528,13 +532,27 @@ public void testFlushAfterRelocation() throws Exception { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } - // Verify segment replication event never happened on replica shard + final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); + + // Verify segment replication event never happened on replica shard other than recovery. + assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setPreference("_only_local").setSize(0).get(), 0); + assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setPreference("_only_local").setSize(0).get(), 0); + SegmentReplicationStatsResponse segmentReplicationStatsResponse = client().admin() .indices() .prepareSegmentReplicationStats(INDEX_NAME) .execute() .actionGet(); - assertTrue(segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0).getReplicaStats().isEmpty()); + final Set replicaStats = segmentReplicationStatsResponse.getReplicationStats() + .get(INDEX_NAME) + .get(0) + .getReplicaStats(); + assertEquals( + Set.of(replicaShard.routingEntry().allocationId().getId()), + replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet()) + ); + // the primary still has not refreshed to update its checkpoint, so our replica is not yet behind. + assertEquals(0, replicaStats.stream().findFirst().get().getCheckpointsBehindCount()); // Relocate primary to new primary. When new primary starts it does perform a flush. logger.info("--> relocate the shard from primary to newPrimary"); diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index 4d6cffa58510f..19fe9ee97cd2f 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -247,7 +247,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L private final Consumer onReplicationGroupUpdated; - private volatile ReplicationCheckpoint lastPublishedReplicationCheckpoint; + private volatile ReplicationCheckpoint latestReplicationCheckpoint; /** * Get all retention leases tracked on this shard. @@ -1054,6 +1054,7 @@ public ReplicationTracker( this.fileBasedRecoveryThreshold = IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.get(indexSettings.getSettings()); this.safeCommitInfoSupplier = safeCommitInfoSupplier; this.onReplicationGroupUpdated = onReplicationGroupUpdated; + this.latestReplicationCheckpoint = indexSettings.isSegRepEnabled() ? ReplicationCheckpoint.empty(shardId) : null; assert Version.V_EMPTY.equals(indexSettings.getIndexVersionCreated()) == false; assert invariant(); } @@ -1212,26 +1213,42 @@ public synchronized void updateVisibleCheckpointForShard(final String allocation */ public synchronized void setLatestReplicationCheckpoint(ReplicationCheckpoint checkpoint) { assert indexSettings.isSegRepEnabled(); - assert handoffInProgress == false; - if (checkpoint.equals(lastPublishedReplicationCheckpoint) == false) { - this.lastPublishedReplicationCheckpoint = checkpoint; - for (Map.Entry entry : checkpoints.entrySet()) { - if (entry.getKey().equals(this.shardAllocationId) == false) { - final CheckpointState cps = entry.getValue(); - if (cps.inSync) { - cps.checkpointTimers.computeIfAbsent(checkpoint, ignored -> { - final ReplicationTimer replicationTimer = new ReplicationTimer(); - replicationTimer.start(); - return replicationTimer; - }); - logger.trace( - () -> new ParameterizedMessage( - "updated last published checkpoint to {} - timers [{}]", - checkpoint, - cps.checkpointTimers.keySet() - ) - ); - } + if (checkpoint.equals(latestReplicationCheckpoint) == false) { + this.latestReplicationCheckpoint = checkpoint; + } + if (primaryMode) { + startReplicationLagTimers(); + } + } + + public ReplicationCheckpoint getLatestReplicationCheckpoint() { + return this.latestReplicationCheckpoint; + } + + private void startReplicationLagTimers() { + for (Map.Entry entry : checkpoints.entrySet()) { + final String allocationId = entry.getKey(); + if (allocationId.equals(this.shardAllocationId) == false) { + final CheckpointState cps = entry.getValue(); + // if the shard is in checkpoints but is unavailable or out of sync we will not track its replication state. + // it is possible for a shard to be in-sync but not yet removed from the checkpoints collection after a failover event. + if (cps.inSync + && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false + && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)) { + cps.checkpointTimers.computeIfAbsent(latestReplicationCheckpoint, ignored -> { + final ReplicationTimer replicationTimer = new ReplicationTimer(); + replicationTimer.start(); + return replicationTimer; + }); + logger.trace( + () -> new ParameterizedMessage( + "updated last published checkpoint for {} at visible cp {} to {} - timers [{}]", + allocationId, + cps.visibleReplicationCheckpoint, + latestReplicationCheckpoint, + cps.checkpointTimers.keySet() + ) + ); } } } @@ -1244,12 +1261,17 @@ public synchronized void setLatestReplicationCheckpoint(ReplicationCheckpoint ch */ public synchronized Set getSegmentReplicationStats() { assert indexSettings.isSegRepEnabled(); - final ReplicationCheckpoint lastPublishedCheckpoint = this.lastPublishedReplicationCheckpoint; - if (primaryMode && lastPublishedCheckpoint != null) { + if (primaryMode) { return this.checkpoints.entrySet() .stream() - .filter(entry -> entry.getKey().equals(this.shardAllocationId) == false && entry.getValue().inSync) - .map(entry -> buildShardStats(lastPublishedCheckpoint.getLength(), entry.getKey(), entry.getValue())) + // filter out this shard's allocation id, any shards that are out of sync or unavailable (shard marked in-sync but has not + // been assigned to a node). + .filter( + entry -> entry.getKey().equals(this.shardAllocationId) == false + && entry.getValue().inSync + && replicationGroup.getUnavailableInSyncShards().contains(entry.getKey()) == false + ) + .map(entry -> buildShardStats(latestReplicationCheckpoint.getLength(), entry.getKey(), entry.getValue())) .collect(Collectors.toUnmodifiableSet()); } return Collections.emptySet(); diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java index da01023ace47c..7549f3450e7f2 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java @@ -33,6 +33,10 @@ public class ReplicationCheckpoint implements Writeable, Comparable tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync)); - // get insync ids, filter out the primary. - final Set inSyncAllocationIds = tracker.getReplicationGroup() - .getInSyncAllocationIds() - .stream() - .filter(id -> tracker.shardAllocationId.equals(id) == false) - .collect(Collectors.toSet()); + + initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED)); final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint( tracker.shardId(), @@ -1831,8 +1827,10 @@ public void testSegmentReplicationCheckpointTracking() { tracker.setLatestReplicationCheckpoint(secondCheckpoint); tracker.setLatestReplicationCheckpoint(thirdCheckpoint); + final Set expectedIds = ids(initializingIds); + Set groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + assertEquals(expectedIds.size(), groupStats.size()); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(3, shardStat.getCheckpointsBehindCount()); assertEquals(100L, shardStat.getBytesBehindCount()); @@ -1840,7 +1838,7 @@ public void testSegmentReplicationCheckpointTracking() { // simulate replicas moved up to date. final Map checkpoints = tracker.checkpoints; - for (String id : inSyncAllocationIds) { + for (String id : expectedIds) { final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id); assertEquals(3, checkpointState.checkpointTimers.size()); tracker.updateVisibleCheckpointForShard(id, initialCheckpoint); @@ -1848,13 +1846,13 @@ public void testSegmentReplicationCheckpointTracking() { } groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + assertEquals(expectedIds.size(), groupStats.size()); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(2, shardStat.getCheckpointsBehindCount()); assertEquals(99L, shardStat.getBytesBehindCount()); } - for (String id : inSyncAllocationIds) { + for (String id : expectedIds) { final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id); assertEquals(2, checkpointState.checkpointTimers.size()); tracker.updateVisibleCheckpointForShard(id, thirdCheckpoint); @@ -1862,7 +1860,7 @@ public void testSegmentReplicationCheckpointTracking() { } groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + assertEquals(expectedIds.size(), groupStats.size()); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(0, shardStat.getCheckpointsBehindCount()); assertEquals(0L, shardStat.getBytesBehindCount()); @@ -1883,19 +1881,24 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() { AllocationId primaryId = activeAllocationIds.iterator().next(); IndexShardRoutingTable routingTable = routingTable(initializingIds, primaryId); final ReplicationTracker tracker = newTracker(primaryId, settings); - tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable); tracker.activatePrimaryMode(NO_OPS_PERFORMED); - assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds))); - assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable)); - assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync)); - // get insync ids, filter out the primary. - final Set inSyncAllocationIds = tracker.getReplicationGroup() - .getInSyncAllocationIds() - .stream() - .filter(id -> tracker.shardAllocationId.equals(id) == false) - .collect(Collectors.toSet()); + initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED)); + + assertEquals(tracker.getReplicationGroup().getRoutingTable(), routingTable); + assertEquals( + "All active & initializing ids are now marked in-sync", + Sets.union(ids(activeAllocationIds), ids(initializingIds)), + tracker.getReplicationGroup().getInSyncAllocationIds() + ); + + assertEquals( + "Active ids are in-sync but still unavailable", + tracker.getReplicationGroup().getUnavailableInSyncShards(), + Sets.difference(ids(activeAllocationIds), Set.of(primaryId.getId())) + ); + assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync)); final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint( tracker.shardId(), @@ -1907,15 +1910,20 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() { ); tracker.setLatestReplicationCheckpoint(initialCheckpoint); + // we expect that the only returned ids from getSegmentReplicationStats will be the initializing ids we marked with + // markAsTrackingAndInSyncQuietly. + // This is because the ids marked active initially are still unavailable (don't have an associated routing entry). + final Set expectedIds = ids(initializingIds); Set groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + final Set actualIds = groupStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet()); + assertEquals(expectedIds, actualIds); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(1, shardStat.getCheckpointsBehindCount()); } // simulate replicas moved up to date. final Map checkpoints = tracker.checkpoints; - for (String id : inSyncAllocationIds) { + for (String id : expectedIds) { final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id); assertEquals(1, checkpointState.checkpointTimers.size()); tracker.updateVisibleCheckpointForShard(id, initialCheckpoint); From 8de5e1ce511e7ed6fa13c823062529ef333657a3 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 25 Jul 2023 12:39:18 -0400 Subject: [PATCH 158/186] OpenJDK Update (July 2023 Patch releases) (#8868) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../java/org/opensearch/gradle/test/DistroTestPlugin.java | 4 ++-- buildSrc/version.properties | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bc645ed1bd6a..68893592ea8d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) - Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) +- OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 7e0dc72cb9108..d32172758cfce 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -75,9 +75,9 @@ import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "11.0.19+7"; + private static final String SYSTEM_JDK_VERSION = "11.0.20+8"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "17.0.7+7"; + private static final String GRADLE_JDK_VERSION = "17.0.8+7"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 246f6e3444224..69aa4c9667c9c 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -2,7 +2,7 @@ opensearch = 3.0.0 lucene = 9.7.0 bundled_jdk_vendor = adoptium -bundled_jdk = 20.0.1+9 +bundled_jdk = 20.0.2+9 # See please https://github.com/adoptium/temurin-build/issues/3371 bundled_jdk_linux_ppc64le = 20+36 From f6078707b9ae3be2ecbf51e45ac21112cf76245d Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 25 Jul 2023 13:43:46 -0400 Subject: [PATCH 159/186] Update Apache Lucene to 9.8.0-snapshot-4373c3b (#8668) Signed-off-by: Andriy Redko --- buildSrc/version.properties | 2 +- libs/core/licenses/lucene-core-9.7.0.jar.sha1 | 1 - .../lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../core/src/main/java/org/opensearch/Version.java | 4 ++-- .../licenses/lucene-expressions-9.7.0.jar.sha1 | 1 - ...ene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../licenses/lucene-analysis-icu-9.7.0.jar.sha1 | 1 - ...ne-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.7.0.jar.sha1 | 1 - ...alysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../licenses/lucene-analysis-nori-9.7.0.jar.sha1 | 1 - ...e-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.7.0.jar.sha1 | 1 - ...alysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.7.0.jar.sha1 | 1 - ...nalysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../lucene-analysis-stempel-9.7.0.jar.sha1 | 1 - ...nalysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../lucene-analysis-morfologik-9.7.0.jar.sha1 | 1 - ...ysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../licenses/lucene-analysis-common-9.7.0.jar.sha1 | 1 - ...analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../licenses/lucene-backward-codecs-9.7.0.jar.sha1 | 1 - ...backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-core-9.7.0.jar.sha1 | 1 - .../lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-grouping-9.7.0.jar.sha1 | 1 - ...lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-highlighter-9.7.0.jar.sha1 | 1 - ...ene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-join-9.7.0.jar.sha1 | 1 - .../lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-memory-9.7.0.jar.sha1 | 1 - .../lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-misc-9.7.0.jar.sha1 | 1 - .../lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-queries-9.7.0.jar.sha1 | 1 - .../lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-queryparser-9.7.0.jar.sha1 | 1 - ...ene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.7.0.jar.sha1 | 1 - .../lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../licenses/lucene-spatial-extras-9.7.0.jar.sha1 | 1 - ...-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-spatial3d-9.7.0.jar.sha1 | 1 - ...ucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + server/licenses/lucene-suggest-9.7.0.jar.sha1 | 1 - .../lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 | 1 + .../search/functionscore/ExplainableScriptIT.java | 14 ++++++++++++-- .../suggest/completion/CompletionSuggester.java | 1 + 50 files changed, 39 insertions(+), 28 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.7.0.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.7.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 69aa4c9667c9c..e4a9293c59b8f 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.7.0 +lucene = 9.8.0-snapshot-4373c3b bundled_jdk_vendor = adoptium bundled_jdk = 20.0.2+9 diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1 deleted file mode 100644 index 2b0f77275c0ab..0000000000000 --- a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad391210ffd806931334be9670a35af00c56f959 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..dc363f2776429 --- /dev/null +++ b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +30c3afcf058532d3d2b8820375043000e7f34a9b \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index f85da63bdbb1f..9329f221922ea 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -55,7 +55,7 @@ public class Version implements Comparable, ToXContentFragment { * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99 * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id - * + *>> * IMPORTANT: Unreleased vs. Released Versions * * All listed versions MUST be released versions, except the last major, the last minor and the last revison. ONLY those are required @@ -91,7 +91,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 deleted file mode 100644 index ecf696b4b3b83..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -297e1cfade4ef71466cc9d4f361d81807c8dc4c8 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..6eaa40708e4ae --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +9f8a34fc3d450343ab05ccb5af318a836a6a5fb3 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 deleted file mode 100644 index 0ed030926ab93..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94293b169fb8572f440a5a4a523320ecf9778ffe \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..8a3332c950b6d --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +fde64e3b23bc9a0849b9897febfe9f13c5113143 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 deleted file mode 100644 index ddd67276606a5..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2df800a38b64867b8dcd61fc2cd986114e4a80cb \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..33c2afacf2395 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +b01a791705fa01fce48dd02ea79fa8045de8dd5e \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 deleted file mode 100644 index 0cd68af98e724..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a01e8153f34d72e8c8c0180c1dea5b10f677dd3a \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..1e7986dafa11e --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +43d19320b1b9cd18638b1602fa87d5f21ee043bc \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 deleted file mode 100644 index c7b4d2dc6da75..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7d47d54683b0b1e09b271c32d1b7d3eb1990f49 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..14880d9c2d243 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +9244dc232f175010b480d4d88e13945c17a0b28b \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 deleted file mode 100644 index 8df7245044171..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..edc4de3fffe28 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +3101a4f79820c1ca3dfb8f49b74c5fb5b32940e1 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 deleted file mode 100644 index 974e4202f5ffb..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d23b1f05b471e05d0d6068b3ece7c8c65672eae7 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..54c310277b09b --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +f12b2a22cd5ebcd84f40a40e78fdd4e268b3b26d \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 deleted file mode 100644 index dce408a7d40ef..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfb4313f3c68d337310522840d7144c1605d084a \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..358db9ea3f0f5 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +7dbf5cc3dff93cc1ffe45d79b129859590d001dd \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 deleted file mode 100644 index 45d8f459573b1..0000000000000 --- a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -27ba6caaa4587a982cd451f7217b5a982bcfc44a \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..e7c7dc2bbc046 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +1446b7641743a1082b566179d1bf2960f5a0724b \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 deleted file mode 100644 index 3981ea4fa226e..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6389463bfbfcf902c8d31d12e9513a6818ac9d5e \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..d0f64519cd6ff --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +127032ea137d2501b24f0e35e5f9a2e1c7864633 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1 deleted file mode 100644 index 2b0f77275c0ab..0000000000000 --- a/server/licenses/lucene-core-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad391210ffd806931334be9670a35af00c56f959 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..dc363f2776429 --- /dev/null +++ b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +30c3afcf058532d3d2b8820375043000e7f34a9b \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1 deleted file mode 100644 index 90acbf6dcee8d..0000000000000 --- a/server/licenses/lucene-grouping-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e6f0c229f4861be641047c33b05067176e4279c \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..294beba43f62a --- /dev/null +++ b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +e6f742efe0ef3b383468fe38f88ab2dd69ed3d2c \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1 deleted file mode 100644 index bfcca0bc6cb5b..0000000000000 --- a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -facb7c7ee0f75ed457a2d98f10d6430e25a53691 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..c2a2ef5b13946 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +3162856444777130dee2c4cabe1bf6d18710ff63 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1 deleted file mode 100644 index 0dab3a7ddc41a..0000000000000 --- a/server/licenses/lucene-join-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d041bdc0947a14223cf68357407ee18b21027587 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..7c6adaaba9cf1 --- /dev/null +++ b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +5fe8383516eca7300f978ce38042e327b0a57877 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1 deleted file mode 100644 index 357a9c4b2ea26..0000000000000 --- a/server/licenses/lucene-memory-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0fade51ee353e15ddbbc45262aafe6f99ed020f1 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..586702c968a77 --- /dev/null +++ b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +b3e77970485be6d2dd59b999bbaa65a2cb993744 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1 deleted file mode 100644 index da5e1921626b2..0000000000000 --- a/server/licenses/lucene-misc-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fcf451e2376526c3a027958812866cc5b0ff13f \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..493598eefff5e --- /dev/null +++ b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +86d667ea2f7fb2142d2acacf801dcea47d014a5e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1 deleted file mode 100644 index fa82e95a7e19f..0000000000000 --- a/server/licenses/lucene-queries-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -126989d4622419aa06fcbf3a342e859cab8c8799 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..1bf937f10d795 --- /dev/null +++ b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +930d004de698f374da8ac5530fd80e241edeba45 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1 deleted file mode 100644 index 438db0aea66e1..0000000000000 --- a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e77bde908ff698354e4a2149e6dd4658b56d7b0 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..74458bc93f90b --- /dev/null +++ b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +f62882823d5aa9ed4cf0081a8c18f35e21992080 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1 deleted file mode 100644 index 38b0b1cccbc29..0000000000000 --- a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9f3e8e1947f2f1c5784132444af51a060ff0b4bf \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..3231d0e067940 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +f1ec1527e283b423b7ff5e12cd8d889e7247199d \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 deleted file mode 100644 index 48679df469fd1..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01b0bc7a407d8c35a70a1adf7966bb3e7caae928 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..dd47faf91f206 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +de787c052879893e47d21fa161c93413665d55d7 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 deleted file mode 100644 index 55d4d217fa6b9..0000000000000 --- a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7c6b1b6e0a70c9cd177371e648648c2f896742a2 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..2b378438bfb14 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +7e541ed960a571f5d9a0ecff5c26fd5ca857581e \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1 deleted file mode 100644 index d4d7e6cd6bed9..0000000000000 --- a/server/licenses/lucene-suggest-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c37fd9a5d71dc87fe1cd4c18ff295ec8cfac170 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..1e3ed6561e3ef --- /dev/null +++ b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +4b222ef09a5f20896d031a8322f2e69304c16384 \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java index f67b913a75871..3651a7354e5de 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java @@ -179,8 +179,18 @@ public void testExplainScript() throws InterruptedException, IOException, Execut for (SearchHit hit : hits.getHits()) { assertThat(hit.getId(), equalTo(Integer.toString(idCounter))); assertThat(hit.getExplanation().toString(), containsString(Double.toString(idCounter))); - assertThat(hit.getExplanation().toString(), containsString("1 = n")); - assertThat(hit.getExplanation().toString(), containsString("1 = N")); + + // Since Apache Lucene 9.8, the scores are not computed because script (see please ExplainableScriptPlugin) + // says "needs_score() == false" + // 19.0 = min of: + // 19.0 = This script returned 19.0 + // 0.0 = _score: + // 0.0 = weight(text:text in 0) [PerFieldSimilarity], result of: + // 0.0 = score(freq=1.0), with freq of: + // 1.0 = freq, occurrences of term within document + // 3.4028235E38 = maxBoost + + assertThat(hit.getExplanation().toString(), containsString("1.0 = freq, occurrences of term within document")); assertThat(hit.getExplanation().getDetails().length, equalTo(2)); idCounter--; } diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java index c7498635d2553..175503cb94e3d 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java @@ -116,6 +116,7 @@ private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSu } } } + collector.finish(); } @Override From b9b5e5c676150f1148c2eb9306e3ce1c8a57c87c Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Tue, 25 Jul 2023 11:02:32 -0700 Subject: [PATCH 160/186] Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory (#8807) * Remove flakey assertion in SearchTimeoutIT Signed-off-by: Jay Deng * Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory Signed-off-by: Jay Deng --------- Signed-off-by: Jay Deng --- CHANGELOG.md | 1 + .../java/org/opensearch/search/SearchTimeoutIT.java | 1 - .../bucket/terms/SignificantTextAggregatorFactory.java | 6 +++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68893592ea8d4..a4c1d8a693df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) - Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) - [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636)) +- Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java index c7392b260319a..aa8ef3f29c989 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java @@ -79,7 +79,6 @@ public void testSimpleTimeout() throws Exception { .get(); assertTrue(searchResponse.isTimedOut()); assertEquals(0, searchResponse.getFailedShards()); - assertTrue(numDocs > searchResponse.getHits().getTotalHits().value); } public void testSimpleDoesNotTimeout() throws Exception { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java index e5cc3f9dbaabd..7f5804c8b9561 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java @@ -148,7 +148,6 @@ protected Aggregator createInternal( : includeExclude.convertToStringFilter(DocValueFormat.RAW, maxRegexLength); MapStringTermsAggregator.CollectorSource collectorSource = new SignificantTextCollectorSource( - queryShardContext.lookup().source(), queryShardContext.bigArrays(), fieldType, sourceFieldNames, @@ -186,13 +185,14 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg private ObjectArray dupSequenceSpotters; SignificantTextCollectorSource( - SourceLookup sourceLookup, BigArrays bigArrays, MappedFieldType fieldType, String[] sourceFieldNames, boolean filterDuplicateText ) { - this.sourceLookup = sourceLookup; + // Create a new SourceLookup instance per aggregator instead of use the shared one from SearchLookup. This is fine because it + // will only be accessed by this Aggregator instance and not anywhere else. + this.sourceLookup = new SourceLookup(); this.bigArrays = bigArrays; this.fieldType = fieldType; this.sourceFieldNames = sourceFieldNames; From 6786608a87d3e7fa306382a38e5702ae830f60e4 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Tue, 25 Jul 2023 11:32:32 -0700 Subject: [PATCH 161/186] Bump version of Hadoop dependencies to 3.3.5 (#6995) Signed-off-by: Tianli Feng Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + distribution/src/config/jvm.options | 3 ++ .../SecuredForkJoinWorkerThreadFactory.java | 46 +++++++++++++++++++ plugins/repository-hdfs/build.gradle | 6 ++- .../licenses/hadoop-client-api-3.3.4.jar.sha1 | 1 - .../licenses/hadoop-client-api-3.3.6.jar.sha1 | 1 + .../hadoop-client-runtime-3.3.4.jar.sha1 | 1 - .../hadoop-client-runtime-3.3.6.jar.sha1 | 1 + .../licenses/hadoop-hdfs-3.3.4.jar.sha1 | 1 - .../licenses/hadoop-hdfs-3.3.6.jar.sha1 | 1 + .../hdfs/HdfsClientThreadLeakFilter.java | 5 +- 11 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c1d8a693df0..e661846a8fc72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) - Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) - OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868) +- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index e15afc0f677c3..952110c6c0289 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -82,3 +82,6 @@ ${error.file} # JDK 20+ Incubating Vector Module for SIMD optimizations; # disabling may reduce performance on vector optimized lucene 20:--add-modules=jdk.incubator.vector + +# HDFS ForkJoinPool.common() support by SecurityManager +-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory diff --git a/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java new file mode 100644 index 0000000000000..fe239fea8129e --- /dev/null +++ b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm; + +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.Permission; +import java.security.Permissions; +import java.security.PrivilegedAction; +import java.security.ProtectionDomain; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory; +import java.util.concurrent.ForkJoinWorkerThread; + +public class SecuredForkJoinWorkerThreadFactory implements ForkJoinWorkerThreadFactory { + static AccessControlContext contextWithPermissions(Permission... perms) { + Permissions permissions = new Permissions(); + for (Permission perm : perms) + permissions.add(perm); + return new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, permissions) }); + } + + // ACC for access to the factory + private static final AccessControlContext ACC = contextWithPermissions( + new RuntimePermission("getClassLoader"), + new RuntimePermission("setContextClassLoader"), + new RuntimePermission("modifyThreadGroup"), + new RuntimePermission("modifyThread") + ); + + public final ForkJoinWorkerThread newThread(ForkJoinPool pool) { + return AccessController.doPrivileged(new PrivilegedAction<>() { + public ForkJoinWorkerThread run() { + return new ForkJoinWorkerThread(pool) { + + }; + } + }, ACC); + } +} diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 0b13e18ba3f4d..6626bfccc6662 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -48,7 +48,7 @@ opensearchplugin { } versions << [ - 'hadoop3': '3.3.4' + 'hadoop3': '3.3.6' ] testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs" @@ -440,3 +440,7 @@ thirdPartyAudit { 'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField', ) } + +tasks.withType(JavaForkOptions) { + systemProperty "java.util.concurrent.ForkJoinPool.common.threadFactory", "org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory" +} diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 deleted file mode 100644 index dd79b8a10cebc..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6339a8f7279310c8b1f7ef314b592d8c71ca72ef \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..d99793bc56522 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 @@ -0,0 +1 @@ +12ac6f103a0ff29fce17a078c7c64d25320b6165 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 deleted file mode 100644 index 32d58d1dc501a..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..ea22d763b7bfa --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 @@ -0,0 +1 @@ +81065531e63fccbe85fb04a3274709593fb00d3c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 deleted file mode 100644 index 532d25a44531f..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -036ef2f86dc44410d2bb5d54ce40435d2484d9a5 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..fe60968056eb7 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 @@ -0,0 +1 @@ +ba40aca60f39599d5b1f1d32b35295bfde1f3c8b \ No newline at end of file diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java index b9b0e9e87dd0c..2758bd020e979 100644 --- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java +++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java @@ -43,6 +43,9 @@ * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect * interrupt handling. * + * In Hadoop 3.3.6, the org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap uses ForkJoinPool + * to perform statistics calculation, leaving dangling workers. + * * @see https://issues.apache.org/jira/browse/HADOOP-12829 * @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner" * @see "org.apache.hadoop.fs.FileSystem.Statistics" @@ -53,6 +56,6 @@ public final class HdfsClientThreadLeakFilter implements ThreadFilter { @Override public boolean reject(Thread t) { - return t.getName().equals(OFFENDING_THREAD_NAME); + return t.getName().equals(OFFENDING_THREAD_NAME) || t.getName().startsWith("ForkJoinPool.commonPool-"); } } From 96ce9de3a2ae45a1909cacea1aba93841c68a5c0 Mon Sep 17 00:00:00 2001 From: ccook-ibm <101672260+ccook-ibm@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:22:36 -0700 Subject: [PATCH 162/186] Extend publishXToMavenLocal explicit dependencies to all repo tasks (#8823) * Extend publishXToMavenLocal explicit dependencies to all repo tasks Signed-off-by: Christopher Cook * Simplify expression Signed-off-by: Christopher Cook Signed-off-by: ccook-ibm --------- Signed-off-by: Christopher Cook Signed-off-by: ccook-ibm --- modules/lang-painless/build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 3b3f2a7f9ca38..d7af8621c478a 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -84,8 +84,7 @@ shadowJar { tasks.validateNebulaPom.dependsOn tasks.generatePomFileForShadowPublication tasks.validateShadowPom.dependsOn tasks.generatePomFileForNebulaPublication -tasks.publishNebulaPublicationToMavenLocal.dependsOn tasks.generatePomFileForShadowPublication -tasks.publishShadowPublicationToMavenLocal.dependsOn tasks.generatePomFileForNebulaPublication +tasks.withType(AbstractPublishToMaven)*.dependsOn "generatePomFileForShadowPublication", "generatePomFileForNebulaPublication" tasks.named("dependencyLicenses").configure { mapping from: /asm-.*/, to: 'asm' From 1b5e64a4d91cf47d10584440f16aac7fa92f7318 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Tue, 25 Jul 2023 21:50:53 -0700 Subject: [PATCH 163/186] Skip testPitCreatedOnReplica IT with remote store (#8877) * Skip testPitCreatedOnReplica IT with remote store Signed-off-by: Suraj Singh * Address review comment Signed-off-by: Suraj Singh --------- Signed-off-by: Suraj Singh --- .../opensearch/indices/replication/SegmentReplicationIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 2a7e8e58b2d03..3ab1a2a8564c5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -1177,6 +1177,10 @@ public void testScrollWithOngoingSegmentReplication() throws Exception { } public void testPitCreatedOnReplica() throws Exception { + assumeFalse( + "Skipping the test as it is flaky with remote store. Tracking issue https://github.com/opensearch-project/OpenSearch/issues/8850", + segmentReplicationWithRemoteEnabled() + ); final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); From e1a41255de203bc57c1e29469a9bb5d50fe0d84b Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 26 Jul 2023 01:19:10 -0700 Subject: [PATCH 164/186] Fix testReplicaHasDiffFilesThanPrimary. (#8863) This test is failing in two ways. First it fails when copying segments from the remote store and there is a cksum mismatch. In this case it is not guaranteed the directory implementation will replace the existing file when copying from the store. This change ensures the mismatched file is cleaned up but only if the shard is not serving reads. In that case we fail the shard so it is re-recovered rather than deleting the segment underneath it. This test also fails with a divide by 0 in RemoteStoreRefreshListener. Signed-off-by: Marc Handalian --- .../main/java/org/opensearch/index/shard/IndexShard.java | 8 ++++++++ .../index/shard/RemoteStoreRefreshListener.java | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0d8a54147d65e..8b6d083379fe1 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4808,6 +4808,14 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo return true; } else { logger.warn("Checksum mismatch between local and remote segment file: {}, will override local file", file); + // If there is a checksum mismatch and we are not serving reads it is safe to go ahead and delete the file now. + // Outside of engine resets this method will be invoked during recovery so this is safe. + if (isReadAllowed() == false) { + localDirectory.deleteFile(file); + } else { + // segment conflict with remote store while the shard is serving reads. + failShard("Local copy of segment " + file + " has a different checksum than the version in remote store", null); + } } } catch (NoSuchFileException | FileNotFoundException e) { logger.debug("File {} does not exist in local FS, downloading from remote store", file); diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 3ea8278038ac5..2385b906a7ae5 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -457,11 +457,10 @@ private void updateLocalSizeMapAndTracker(Collection segmentFiles) { private void updateFinalStatusInSegmentTracker(boolean uploadStatus, long bytesBeforeUpload, long startTimeInNS) { if (uploadStatus) { long bytesUploaded = segmentTracker.getUploadBytesSucceeded() - bytesBeforeUpload; - long timeTakenInMS = (System.nanoTime() - startTimeInNS) / 1_000_000L; - + long timeTakenInMS = TimeValue.nsecToMSec(System.nanoTime() - startTimeInNS); segmentTracker.incrementTotalUploadsSucceeded(); segmentTracker.addUploadBytes(bytesUploaded); - segmentTracker.addUploadBytesPerSec((bytesUploaded * 1_000L) / timeTakenInMS); + segmentTracker.addUploadBytesPerSec((bytesUploaded * 1_000L) / Math.max(1, timeTakenInMS)); segmentTracker.addUploadTimeMs(timeTakenInMS); } else { segmentTracker.incrementTotalUploadsFailed(); From 4fd6877eef7c9cb0b48268a7505506aeb33278a2 Mon Sep 17 00:00:00 2001 From: luyuncheng Date: Thu, 27 Jul 2023 00:30:33 +0800 Subject: [PATCH 165/186] Replace the deprecated IndexReader APIs with new storedFields() & termVectors() (#7792) * 1. Remove calling deprecated document api Signed-off-by: luyuncheng * 1. Remove calling deprecated document api 2. Fixed some calling and Tests Signed-off-by: luyuncheng * 1. Remove calling deprecated document api 2. Fixed some calling and Tests 3. Spotless java Signed-off-by: luyuncheng * add changelog Signed-off-by: luyuncheng * add changelog Signed-off-by: luyuncheng * merge main into branch Signed-off-by: luyuncheng * merge main into branch update CHANGELOG.md Signed-off-by: luyuncheng --------- Signed-off-by: luyuncheng --- CHANGELOG.md | 1 + .../common/lucene/search/XMoreLikeThis.java | 8 +++- .../gateway/PersistedClusterStateService.java | 6 +-- .../index/engine/InternalEngine.java | 4 +- .../index/engine/LuceneChangesSnapshot.java | 2 +- .../index/engine/TranslogLeafReader.java | 41 +++++++++++-------- .../opensearch/index/get/ShardGetService.java | 2 +- .../index/shard/ShardSplittingQuery.java | 2 +- .../index/termvectors/TermVectorsService.java | 7 +++- .../opensearch/search/fetch/FetchPhase.java | 2 +- .../subphase/highlight/HighlightUtils.java | 2 +- .../search/lookup/LeafFieldsLookup.java | 2 +- .../search/lookup/SourceLookup.java | 2 +- .../opensearch/common/lucene/LuceneTests.java | 7 +++- .../lucene/index/FreqTermsEnumTests.java | 4 +- .../index/engine/InternalEngineTests.java | 7 +++- .../RecoverySourcePruneMergePolicyTests.java | 10 +++-- .../query/MoreLikeThisQueryBuilderTests.java | 2 +- .../index/shard/RefreshListenersTests.java | 6 +-- .../index/shard/ShardSplittingQueryTests.java | 2 +- .../indices/IndicesRequestCacheTests.java | 2 +- .../lucene/queries/BlendedTermQueryTests.java | 6 +-- .../search/lookup/LeafFieldsLookupTests.java | 15 +++---- .../slice/DocValuesSliceQueryTests.java | 2 +- .../search/slice/TermsSliceQueryTests.java | 2 +- .../index/engine/EngineTestCase.java | 7 +++- 26 files changed, 92 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e661846a8fc72..a84f5138bd112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) +- Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java b/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java index d7ffa2df943b7..49148890abd55 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java @@ -56,7 +56,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; @@ -808,8 +810,10 @@ public String describeParams() { */ private PriorityQueue retrieveTerms(int docNum) throws IOException { Map termFreqMap = new HashMap<>(); + final TermVectors termVectors = ir.termVectors(); + final StoredFields storedFields = ir.storedFields(); for (String fieldName : fieldNames) { - final Fields vectors = ir.getTermVectors(docNum); + final Fields vectors = termVectors.get(docNum); final Terms vector; if (vectors != null) { vector = vectors.terms(fieldName); @@ -819,7 +823,7 @@ private PriorityQueue retrieveTerms(int docNum) throws IOException { // field does not store term vector info if (vector == null) { - Document d = ir.document(docNum); + Document d = storedFields.document(docNum); IndexableField fields[] = d.getFields(fieldName); for (IndexableField field : fields) { final String stringValue = field.stringValue(); diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java index 8940b0ed25ed4..caddf92150cff 100644 --- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java @@ -45,6 +45,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -507,12 +508,11 @@ private static void consumeFromType(IndexSearcher indexSearcher, String type, Ch final Bits liveDocs = leafReaderContext.reader().getLiveDocs(); final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get; final DocIdSetIterator docIdSetIterator = scorer.iterator(); + final StoredFields storedFields = leafReaderContext.reader().storedFields(); while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { if (isLiveDoc.test(docIdSetIterator.docID())) { logger.trace("processing doc {}", docIdSetIterator.docID()); - bytesRefConsumer.accept( - leafReaderContext.reader().document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME) - ); + bytesRefConsumer.accept(storedFields.document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME)); } } } diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 77e2f5cbef7f9..6f8b6d449695e 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -2889,6 +2890,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead final CombinedDocValues dv = new CombinedDocValues(leaf.reader()); final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor(); final DocIdSetIterator iterator = scorer.iterator(); + final StoredFields storedFields = leaf.reader().storedFields(); int docId; while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { final long primaryTerm = dv.docPrimaryTerm(docId); @@ -2896,7 +2898,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead localCheckpointTracker.markSeqNoAsProcessed(seqNo); localCheckpointTracker.markSeqNoAsPersisted(seqNo); idFieldVisitor.reset(); - leaf.reader().document(docId, idFieldVisitor); + storedFields.document(docId, idFieldVisitor); if (idFieldVisitor.getId() == null) { assert dv.isTombstone(docId); continue; diff --git a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java index abde2aff6e9e6..23fe59456887e 100644 --- a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java @@ -289,7 +289,7 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException { ? SourceFieldMapper.RECOVERY_SOURCE_NAME : SourceFieldMapper.NAME; final FieldsVisitor fields = new FieldsVisitor(true, sourceField); - leaf.reader().document(segmentDocID, fields); + leaf.reader().storedFields().document(segmentDocID, fields); final Translog.Operation op; final boolean isTombstone = parallelArray.isTombStone[docIndex]; diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java index 5efafb562df74..417cdd5a8f030 100644 --- a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java +++ b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java @@ -221,28 +221,33 @@ public int maxDoc() { @Override public void document(int docID, StoredFieldVisitor visitor) throws IOException { - if (docID != 0) { - throw new IllegalArgumentException("no such doc ID " + docID); - } - if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) { - assert operation.source().toBytesRef().offset == 0; - assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length; - visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes); - } - if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { - visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); - } - if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { - BytesRef bytesRef = Uid.encodeId(operation.id()); - final byte[] id = new byte[bytesRef.length]; - System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length); - visitor.binaryField(FAKE_ID_FIELD, id); - } + storedFields().document(docID, visitor); } @Override public StoredFields storedFields() throws IOException { - throw new UnsupportedOperationException(); + return new StoredFields() { + @Override + public void document(int docID, StoredFieldVisitor visitor) throws IOException { + if (docID != 0) { + throw new IllegalArgumentException("no such doc ID " + docID); + } + if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) { + assert operation.source().toBytesRef().offset == 0; + assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length; + visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes); + } + if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { + visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); + } + if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { + BytesRef bytesRef = Uid.encodeId(operation.id()); + final byte[] id = new byte[bytesRef.length]; + System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length); + visitor.binaryField(FAKE_ID_FIELD, id); + } + } + }; } @Override diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java index a815074119fb1..b3715e097322d 100644 --- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java @@ -276,7 +276,7 @@ private GetResult innerGetLoadFromStoredFields( ); if (fieldVisitor != null) { try { - docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor); + docIdAndVersion.reader.storedFields().document(docIdAndVersion.docId, fieldVisitor); } catch (IOException e) { throw new OpenSearchException("Failed to get id [" + id + "]", e); } diff --git a/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java index 6d559a6cc0673..219ead931797a 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java @@ -286,7 +286,7 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { boolean matches(int doc) throws IOException { routing = id = null; leftToVisit = 2; - leafReader.document(doc, this); + leafReader.storedFields().document(doc, this); assert id != null : "docID must not be null - we might have hit a nested document"; int targetShardId = OperationRouting.generateShardId(indexMetadata, id, routing); return targetShardId != shardId; diff --git a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java index 0e9d7c2f14113..4d515fadb5a13 100644 --- a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.memory.MemoryIndex; import org.opensearch.OpenSearchException; @@ -127,7 +128,8 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ /* or from an existing document */ else if (docIdAndVersion != null) { // fields with stored term vectors - termVectorsByField = docIdAndVersion.reader.getTermVectors(docIdAndVersion.docId); + TermVectors termVectors = docIdAndVersion.reader.termVectors(); + termVectorsByField = termVectors.get(docIdAndVersion.docId); Set selectedFields = request.selectedFields(); // generate tvs for fields where analyzer is overridden if (selectedFields == null && request.perFieldAnalyzer() != null) { @@ -322,7 +324,8 @@ private static Fields generateTermVectors( } } /* and read vectors from it */ - return index.createSearcher().getIndexReader().getTermVectors(0); + TermVectors termVectors = index.createSearcher().getIndexReader().termVectors(); + return termVectors.get(0); } private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java index 97a6093ea39d1..67c6eeae02271 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java @@ -160,7 +160,7 @@ public void execute(SearchContext context) { SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) currentReaderContext.reader(); fieldReader = lf.getSequentialStoredFieldsReader()::document; } else { - fieldReader = currentReaderContext.reader()::document; + fieldReader = currentReaderContext.reader().storedFields()::document; } for (FetchSubPhaseProcessor processor : processors) { processor.setNextReader(currentReaderContext); diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java index 7a358b7e4b252..2238554a12149 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java @@ -72,7 +72,7 @@ public static List loadFieldValues( ) throws IOException { if (forceSource == false && fieldType.isStored()) { CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(fieldType.name()), false); - hitContext.reader().document(hitContext.docId(), fieldVisitor); + hitContext.reader().storedFields().document(hitContext.docId(), fieldVisitor); List textsToHighlight = fieldVisitor.fields().get(fieldType.name()); return textsToHighlight != null ? textsToHighlight : Collections.emptyList(); } diff --git a/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java b/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java index 007ee49b3e697..47bb8754a5803 100644 --- a/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java @@ -153,7 +153,7 @@ private FieldLookup loadFieldData(String name) { List values = new ArrayList<>(2); SingleFieldsVisitor visitor = new SingleFieldsVisitor(data.fieldType(), values); try { - reader.document(docId, visitor); + reader.storedFields().document(docId, visitor); } catch (IOException e) { throw new OpenSearchParseException("failed to load field [{}]", e, name); } diff --git a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java index 1341fc0fdabb3..00fdca4e143ee 100644 --- a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java @@ -140,7 +140,7 @@ public void setSegmentAndDocument(LeafReaderContext context, int docId) { SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) context.reader(); fieldReader = lf.getSequentialStoredFieldsReader()::document; } else { - fieldReader = context.reader()::document; + fieldReader = context.reader().storedFields()::document; } } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java index ef0bedac72541..f7be2c4876e6f 100644 --- a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; @@ -565,12 +566,13 @@ public void testWrapAllDocsLive() throws Exception { } try (DirectoryReader unwrapped = DirectoryReader.open(writer)) { DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped); + StoredFields storedFields = reader.storedFields(); assertThat(reader.numDocs(), equalTo(liveDocs.size())); IndexSearcher searcher = new IndexSearcher(reader); Set actualDocs = new HashSet<>(); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - actualDocs.add(reader.document(scoreDoc.doc).get("id")); + actualDocs.add(storedFields.document(scoreDoc.doc).get("id")); } assertThat(actualDocs, equalTo(liveDocs)); } @@ -609,13 +611,14 @@ public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception { } try (DirectoryReader unwrapped = DirectoryReader.open(writer)) { DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped); + StoredFields storedFields = reader.storedFields(); assertThat(reader.maxDoc(), equalTo(numDocs + abortedDocs)); assertThat(reader.numDocs(), equalTo(liveDocs.size())); IndexSearcher searcher = new IndexSearcher(reader); List actualDocs = new ArrayList<>(); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - actualDocs.add(reader.document(scoreDoc.doc).get("id")); + actualDocs.add(storedFields.document(scoreDoc.doc).get("id")); } assertThat(actualDocs, equalTo(liveDocs)); } diff --git a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java index 60f39208d008a..9423d3c17c98a 100644 --- a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java @@ -43,6 +43,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.Query; @@ -149,9 +150,10 @@ public void setUp() throws Exception { // now go over each doc, build the relevant references and filter reader = DirectoryReader.open(iw); + StoredFields storedFields = reader.storedFields(); List filterTerms = new ArrayList<>(); for (int docId = 0; docId < reader.maxDoc(); docId++) { - Document doc = reader.document(docId); + Document doc = storedFields.document(docId); addFreqs(doc, referenceAll); if (!deletedIds.contains(doc.getField("id").stringValue())) { addFreqs(doc, referenceNotDeleted); diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 612bebf74bc70..c2ac8b0e1d3b3 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -2541,6 +2541,7 @@ class OpAndVersion { final Term uidTerm = newUid(doc); engine.index(indexForDoc(doc)); final BiFunction searcherFactory = engine::acquireSearcher; + for (int i = 0; i < thread.length; i++) { thread[i] = new Thread(() -> { startGun.countDown(); @@ -2549,10 +2550,12 @@ class OpAndVersion { } catch (InterruptedException e) { throw new AssertionError(e); } + for (int op = 0; op < opsPerThread; op++) { try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) { + FieldsVisitor visitor = new FieldsVisitor(true); - get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); + get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor); List values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString())); String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null; String added = "v_" + idGenerator.incrementAndGet(); @@ -2608,7 +2611,7 @@ class OpAndVersion { try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) { FieldsVisitor visitor = new FieldsVisitor(true); - get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); + get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor); List values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString())); assertThat(currentValues, equalTo(new HashSet<>(values))); } diff --git a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java index a8279ed908779..fed521e2d5ed9 100644 --- a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java +++ b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.MatchAllDocsQuery; @@ -89,8 +90,9 @@ public void testPruneAll() throws IOException { writer.forceMerge(1); writer.commit(); try (DirectoryReader reader = DirectoryReader.open(writer)) { + StoredFields storedFields = reader.storedFields(); for (int i = 0; i < reader.maxDoc(); i++) { - Document document = reader.document(i); + Document document = storedFields.document(i); assertEquals(1, document.getFields().size()); assertEquals("source", document.getFields().get(0).name()); } @@ -157,11 +159,12 @@ public void testPruneSome() throws IOException { writer.forceMerge(1); writer.commit(); try (DirectoryReader reader = DirectoryReader.open(writer)) { + StoredFields storedFields = reader.storedFields(); assertEquals(1, reader.leaves().size()); NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source"); assertNotNull(extra_source); for (int i = 0; i < reader.maxDoc(); i++) { - Document document = reader.document(i); + Document document = storedFields.document(i); Set collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet()); assertTrue(collect.contains("source")); assertTrue(collect.contains("even")); @@ -197,11 +200,12 @@ public void testPruneNone() throws IOException { writer.forceMerge(1); writer.commit(); try (DirectoryReader reader = DirectoryReader.open(writer)) { + StoredFields storedFields = reader.storedFields(); assertEquals(1, reader.leaves().size()); NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source"); assertNotNull(extra_source); for (int i = 0; i < reader.maxDoc(); i++) { - Document document = reader.document(i); + Document document = storedFields.document(i); Set collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet()); assertTrue(collect.contains("source")); assertTrue(collect.contains("extra_source")); diff --git a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java index e6ec132a10474..2d950f0994976 100644 --- a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -268,7 +268,7 @@ private static Fields generateFields(String[] fieldNames, String text) throws IO for (String fieldName : fieldNames) { index.addField(fieldName, text, new WhitespaceAnalyzer()); } - return index.createSearcher().getIndexReader().getTermVectors(0); + return index.createSearcher().getIndexReader().termVectors().get(0); } @Override diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java index f812312004b21..1be6c07539cb7 100644 --- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java @@ -36,6 +36,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; @@ -369,9 +370,8 @@ public void testLotsOfThreads() throws Exception { try (Engine.GetResult getResult = engine.get(get, engine::acquireSearcher)) { assertTrue("document not found", getResult.exists()); assertEquals(iteration, getResult.version()); - org.apache.lucene.document.Document document = getResult.docIdAndVersion().reader.document( - getResult.docIdAndVersion().docId - ); + StoredFields storedFields = getResult.docIdAndVersion().reader.storedFields(); + org.apache.lucene.document.Document document = storedFields.document(getResult.docIdAndVersion().docId); assertThat(document.getValues("test"), arrayContaining(testFieldValue)); } } catch (Exception t) { diff --git a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java index 04dcea210640c..940d9a4ead5f9 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java @@ -326,7 +326,7 @@ void assertSplit(Directory dir, IndexMetadata metadata, int targetShardId, boole } assertEquals(shard_id.docID(), doc); long shardID = shard_id.nextValue(); - BytesRef id = reader.document(doc).getBinaryValue("_id"); + BytesRef id = reader.storedFields().document(doc).getBinaryValue("_id"); String actualId = Uid.decodeId(id.bytes, id.offset, id.length); assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId); } diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index 3508020cb23d7..a06688150a38a 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -353,7 +353,7 @@ public BytesReference get() { IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1); assertEquals(1, topDocs.totalHits.value); - Document document = reader.document(topDocs.scoreDocs[0].doc); + Document document = reader.storedFields().document(topDocs.scoreDocs[0].doc); out.writeString(document.get("value")); loadedFromCache = false; return out.bytes(); diff --git a/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java index bf9aac344e3df..6844742759883 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java @@ -112,7 +112,7 @@ public void testDismaxQuery() throws IOException { query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "generator"), 0.1f), BooleanClause.Occur.SHOULD); TopDocs search = searcher.search(query.build(), 10); ScoreDoc[] scoreDocs = search.scoreDocs; - assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue()); + assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue()); } { BooleanQuery.Builder query = new BooleanQuery.Builder(); @@ -134,7 +134,7 @@ public void testDismaxQuery() throws IOException { query.add(gen, BooleanClause.Occur.SHOULD); TopDocs search = searcher.search(query.build(), 4); ScoreDoc[] scoreDocs = search.scoreDocs; - assertEquals(Integer.toString(1), reader.document(scoreDocs[0].doc).getField("id").stringValue()); + assertEquals(Integer.toString(1), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue()); } { @@ -269,7 +269,7 @@ public void testMinTTF() throws IOException { Query query = BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "foo"), 0.1f); TopDocs search = searcher.search(query, 10); ScoreDoc[] scoreDocs = search.scoreDocs; - assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue()); + assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue()); } reader.close(); w.close(); diff --git a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java index 0155e288a96fd..85aacfbd63ee2 100644 --- a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java +++ b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java @@ -36,6 +36,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.index.mapper.MappedFieldType; @@ -43,11 +44,11 @@ import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; +import java.io.IOException; import java.util.Collections; import java.util.List; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -88,12 +89,12 @@ public void setUp() throws Exception { ); LeafReader leafReader = mock(LeafReader.class); - doAnswer(invocation -> { - Object[] args = invocation.getArguments(); - StoredFieldVisitor visitor = (StoredFieldVisitor) args[1]; - visitor.doubleField(mockFieldInfo, 2.718); - return null; - }).when(leafReader).document(anyInt(), any(StoredFieldVisitor.class)); + doAnswer(invocation -> new StoredFields() { + @Override + public void document(int docID, StoredFieldVisitor visitor) throws IOException { + visitor.doubleField(mockFieldInfo, 2.718); + } + }).when(leafReader).storedFields(); fieldsLookup = new LeafFieldsLookup(mapperService, leafReader); } diff --git a/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java b/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java index 86dc27d73d21f..4d2a441a180a6 100644 --- a/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java +++ b/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java @@ -110,7 +110,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { - Document d = context.reader().document(doc, Collections.singleton("uuid")); + Document d = context.reader().storedFields().document(doc, Collections.singleton("uuid")); String uuid = d.get("uuid"); assertThat(keys.contains(uuid), equalTo(true)); keys.remove(uuid); diff --git a/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java b/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java index 84a65b3c8a7cb..8efa4eeef80dd 100644 --- a/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java +++ b/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java @@ -120,7 +120,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { - Document d = context.reader().document(doc, Collections.singleton("uuid")); + Document d = context.reader().storedFields().document(doc, Collections.singleton("uuid")); String uuid = d.get("uuid"); assertThat(keys.contains(uuid), equalTo(true)); keys.remove(uuid); diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index 9c6f36c87896c..1ac92bbb479c3 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.LiveIndexWriterConfig; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -1302,6 +1303,7 @@ public static List getDocIds(Engine engine, boolean refresh NumericDocValues primaryTermDocValues = reader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); NumericDocValues versionDocValues = reader.getNumericDocValues(VersionFieldMapper.NAME); Bits liveDocs = reader.getLiveDocs(); + StoredFields storedFields = reader.storedFields(); for (int i = 0; i < reader.maxDoc(); i++) { if (liveDocs == null || liveDocs.get(i)) { if (primaryTermDocValues.advanceExact(i) == false) { @@ -1309,7 +1311,7 @@ public static List getDocIds(Engine engine, boolean refresh continue; } final long primaryTerm = primaryTermDocValues.longValue(); - Document doc = reader.document(i, Sets.newHashSet(IdFieldMapper.NAME, SourceFieldMapper.NAME)); + Document doc = storedFields.document(i, Sets.newHashSet(IdFieldMapper.NAME, SourceFieldMapper.NAME)); BytesRef binaryID = doc.getBinaryValue(IdFieldMapper.NAME); String id = Uid.decodeId(Arrays.copyOfRange(binaryID.bytes, binaryID.offset, binaryID.offset + binaryID.length)); final BytesRef source = doc.getBinaryValue(SourceFieldMapper.NAME); @@ -1463,6 +1465,7 @@ public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings for (LeafReaderContext leaf : wrappedReader.leaves()) { NumericDocValues primaryTermDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); NumericDocValues seqNoDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME); + final StoredFields storedFields = leaf.reader().storedFields(); int docId; while ((docId = seqNoDocValues.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(seqNoDocValues.advanceExact(docId)); @@ -1471,7 +1474,7 @@ public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings if (primaryTermDocValues.advanceExact(docId)) { if (seqNos.add(seqNo) == false) { final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor(); - leaf.reader().document(docId, idFieldVisitor); + storedFields.document(docId, idFieldVisitor); throw new AssertionError("found multiple documents for seq=" + seqNo + " id=" + idFieldVisitor.getId()); } } From 4319f2b571c82d7ec5608ffdc31ca4105961d064 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 26 Jul 2023 10:23:15 -0700 Subject: [PATCH 166/186] Remove unnecessary refresh listeners from NRTReplicationReaderManager. (#8859) * Remove unnecessary refresh listeners from NRTReplicationReaderManager. This change removes RefreshListeners used by InternalEngine to provide waitFor functionality. These listeners were previously registered onto NRT replicas only to be force released on the next refresh cycle without actually refreshing the reader. This change also removes the unnecessary blocking refresh from NRTReaderManager because we no longer have conflicting refresh invocations from scheduledRefresh. Signed-off-by: Marc Handalian * Reduce the amount of docs ingested with testPrimaryRelocation and testPrimaryRelocationWithSegRepFailure. These tests were ingesting 100-1k docs and randomly selecting a refresh policy. Wtih the IMMEDIATE refresh policy a blocking refresh is performed that increase the time required for the primary to block operations for relocation. On my machine this change reduces the test time with max docs from 1m to 5-6s. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../SegmentReplicationRelocationIT.java | 4 ++-- .../index/engine/NRTReplicationEngine.java | 21 ++++++------------- .../engine/NRTReplicationReaderManager.java | 2 +- .../opensearch/index/shard/IndexShard.java | 3 ++- .../SegmentReplicationIndexShardTests.java | 6 ++++++ 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java index 7cf7e5148dd4a..3024eeb798b48 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java @@ -60,7 +60,7 @@ public void testPrimaryRelocation() throws Exception { createIndex(1); final String replica = internalCluster().startNode(); ensureGreen(INDEX_NAME); - final int initialDocCount = scaledRandomIntBetween(100, 1000); + final int initialDocCount = scaledRandomIntBetween(10, 100); final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); final List> pendingIndexResponses = new ArrayList<>(); for (int i = 0; i < initialDocCount; i++) { @@ -137,7 +137,7 @@ public void testPrimaryRelocationWithSegRepFailure() throws Exception { createIndex(1); final String replica = internalCluster().startNode(); ensureGreen(INDEX_NAME); - final int initialDocCount = scaledRandomIntBetween(100, 1000); + final int initialDocCount = scaledRandomIntBetween(10, 100); final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); final List> pendingIndexResponses = new ArrayList<>(); for (int i = 0; i < initialDocCount; i++) { diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index a9f7a2e70884c..b55508b7facd3 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -77,9 +77,10 @@ public NRTReplicationEngine(EngineConfig engineConfig) { this.completionStatsCache = new CompletionStatsCache(() -> acquireSearcher("completion_stats")); this.readerManager = readerManager; this.readerManager.addListener(completionStatsCache); - for (ReferenceManager.RefreshListener listener : engineConfig.getExternalRefreshListener()) { - this.readerManager.addListener(listener); - } + // NRT Replicas do not have a concept of Internal vs External reader managers. + // We also do not want to wire up refresh listeners for waitFor & pending refresh location. + // which are the current external listeners set from IndexShard. + // Only wire up the internal listeners. for (ReferenceManager.RefreshListener listener : engineConfig.getInternalRefreshListener()) { this.readerManager.addListener(listener); } @@ -322,22 +323,12 @@ public List segments(boolean verbose) { @Override public void refresh(String source) throws EngineException { - maybeRefresh(source); + // Refresh on this engine should only ever happen in the reader after new segments arrive. } @Override public boolean maybeRefresh(String source) throws EngineException { - ensureOpen(); - try { - return readerManager.maybeRefresh(); - } catch (IOException e) { - try { - failEngine("refresh failed source[" + source + "]", e); - } catch (Exception inner) { - e.addSuppressed(inner); - } - throw new RefreshFailedEngineException(shardId, e); - } + return false; } @Override diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java index 35409437f605a..7b4c93c7235fe 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java @@ -103,7 +103,7 @@ public void updateSegments(SegmentInfos infos) throws IOException { // is always increased. infos.updateGeneration(currentInfos); currentInfos = infos; - maybeRefreshBlocking(); + maybeRefresh(); } public SegmentInfos getSegmentInfos() { diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 8b6d083379fe1..e43b9773cc1e0 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4423,7 +4423,8 @@ public void addRefreshListener(Translog.Location location, Consumer lis readAllowed = isReadAllowed(); } } - if (readAllowed) { + // NRT Replicas will not accept refresh listeners. + if (readAllowed && isSegmentReplicationAllowed() == false) { refreshListeners.addOrNotify(location, listener); } else { // we're not yet ready fo ready for reads, just ignore refresh cycles diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index d988e34ef18dc..0c68512f93ea6 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -120,6 +120,12 @@ public void testReplicationCheckpointNotNullForSegRep() throws IOException { closeShards(indexShard); } + public void testNRTReplicasDoNotAcceptRefreshListeners() throws IOException { + final IndexShard indexShard = newStartedShard(false, settings, new NRTReplicationEngineFactory()); + indexShard.addRefreshListener(mock(Translog.Location.class), Assert::assertFalse); + closeShards(indexShard); + } + public void testSegmentInfosAndReplicationCheckpointTuple() throws Exception { try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { shards.startAll(); From 99f28cb6f6bd73a6d832c6bbb88e5b9c2ccdb139 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 26 Jul 2023 11:17:22 -0700 Subject: [PATCH 167/186] Updated Version.java path from server to libs in version.yml (#8883) * Updating Version.java from server/ to buildSrc/ Signed-off-by: Varun Jain * Adding Changelog Signed-off-by: Varun Jain * Path update Signed-off-by: Varun Jain * Changelog Update Signed-off-by: Varun Jain * Removing Changelog from commit Signed-off-by: Varun Jain --------- Signed-off-by: Varun Jain --- .github/workflows/version.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index fdf42a9a2731e..764a365e7411c 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -54,8 +54,8 @@ jobs: echo " - \"$CURRENT_VERSION\"" >> .ci/bwcVersions sed -i "s/opensearch = $CURRENT_VERSION/opensearch = $NEXT_VERSION/g" buildSrc/version.properties echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE - sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java - sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" server/src/main/java/org/opensearch/Version.java + sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java + sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request uses: peter-evans/create-pull-request@v3 @@ -82,7 +82,7 @@ jobs: echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION sed -i "s/- \"$CURRENT_VERSION\"/\0\n - \"$NEXT_VERSION\"/g" .ci/bwcVersions echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE - sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java + sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request uses: peter-evans/create-pull-request@v3 From c25c175c94efa4d90a1d98e3c7eac3e4aad6e41f Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 27 Jul 2023 08:34:04 +0530 Subject: [PATCH 168/186] [Remote Store] Add support to restore only unassigned shards of an index (#8792) * Add support to restore only unassigned shards of an index --------- Signed-off-by: Sachin Kale Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- CHANGELOG.md | 1 + .../RemoteStoreBaseIntegTestCase.java | 10 +- .../remotestore/RemoteStoreForceMergeIT.java | 12 +- .../opensearch/remotestore/RemoteStoreIT.java | 140 ++++++++++++------ .../snapshots/RestoreSnapshotIT.java | 14 +- .../restore/RestoreRemoteStoreRequest.java | 33 ++++- .../cluster/routing/IndexRoutingTable.java | 12 +- .../cluster/routing/RoutingTable.java | 8 +- .../cluster/RestRestoreRemoteStoreAction.java | 1 + .../opensearch/snapshots/RestoreService.java | 44 ++++-- .../RestoreRemoteStoreRequestTests.java | 2 + .../cluster/routing/RoutingTableTests.java | 34 ++++- .../SegmentReplicationIndexShardTests.java | 3 - 13 files changed, 236 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a84f5138bd112..12dae4fca545e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) +- [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792)) - Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index c5d023bdd7a64..2887fbc56106c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -26,6 +26,7 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -37,6 +38,13 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final int REPLICA_COUNT = 1; protected Path absolutePath; protected Path absolutePath2; + private final List documentKeys = List.of( + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5) + ); @Override protected boolean addMockInternalEngine() { @@ -59,7 +67,7 @@ public Settings indexSettings() { IndexResponse indexSingleDoc(String indexName) { return client().prepareIndex(indexName) .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) + .setSource(documentKeys.get(randomIntBetween(0, documentKeys.size() - 1)), randomAlphaOfLength(5)) .get(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index b4456f887cbaa..4d5648c74ba5c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -104,9 +104,17 @@ private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlus Map indexStats = indexData(numberOfIterations, invokeFlush, flushAfterMerge, deletedDocs); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + } + client().admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(INDEX_NAME); if (deletedDocs == -1) { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 42bd4b5173fa3..693c4113f8f3b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -11,6 +11,7 @@ import org.hamcrest.MatcherAssert; import org.junit.Before; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.index.IndexResponse; @@ -18,7 +19,6 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RecoverySource; -import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.RemoteStoreRefreshListener; import org.opensearch.indices.recovery.RecoveryState; @@ -34,15 +34,16 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import static org.hamcrest.Matchers.comparesEqualTo; -import static org.hamcrest.Matchers.oneOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.comparesEqualTo; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0) public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -68,13 +69,6 @@ public Settings indexSettings() { return remoteStoreIndexSettings(0); } - private IndexResponse indexSingleDoc() { - return client().prepareIndex(INDEX_NAME) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - private Map indexData(int numberOfIterations, boolean invokeFlush, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; @@ -93,7 +87,7 @@ private Map indexData(int numberOfIterations, boolean invokeFlush, refreshedOrFlushedOperations = totalOperations; int numberOfOperations = randomIntBetween(20, 50); for (int j = 0; j < numberOfOperations; j++) { - IndexResponse response = INDEX_NAME.equals(index) ? indexSingleDoc() : indexSingleDoc(index); + IndexResponse response = indexSingleDoc(index); maxSeqNo = response.getSeqNo(); shardId = response.getShardId().id(); indexingStats.put(MAX_SEQ_NO_TOTAL + "-shard-" + shardId, maxSeqNo); @@ -109,12 +103,14 @@ private Map indexData(int numberOfIterations, boolean invokeFlush, } private void verifyRestoredData(Map indexStats, boolean checkTotal, String indexName) { + // This is required to get updated number from already active shards which were not restored + refresh(indexName); String statsGranularity = checkTotal ? TOTAL_OPERATIONS : REFRESHED_OR_FLUSHED_OPERATIONS; String maxSeqNoGranularity = checkTotal ? MAX_SEQ_NO_TOTAL : MAX_SEQ_NO_REFRESHED_OR_FLUSHED; ensureYellowAndNoInitializingShards(indexName); ensureGreen(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity)); - IndexResponse response = INDEX_NAME.equals(indexName) ? indexSingleDoc() : indexSingleDoc(indexName); + IndexResponse response = indexSingleDoc(indexName); assertEquals(indexStats.get(maxSeqNoGranularity + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo()); refresh(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); @@ -130,6 +126,28 @@ private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, St } } + private void restore(String... indices) { + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices)); + } + client().admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(indices).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); + } + + private void restoreAndVerify(int shardCount, int replicaCount, Map indexStats) { + restore(INDEX_NAME); + ensureGreen(INDEX_NAME); + // This is required to get updated number from already active shards which were not restored + assertEquals(shardCount * (1 + replicaCount), getNumShards(INDEX_NAME).totalNumShards); + assertEquals(replicaCount, getNumShards(INDEX_NAME).numReplicas); + verifyRestoredData(indexStats, true, INDEX_NAME); + } + /** * Helper function to test restoring an index with no replication from remote store. Only primary node is dropped. * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. @@ -144,23 +162,16 @@ private void testRestoreFlow(int numberOfIterations, boolean invokeFlush, int sh internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); ensureRed(INDEX_NAME); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); - - ensureGreen(INDEX_NAME); - assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); - verifyRestoredData(indexStats, true, INDEX_NAME); + restoreAndVerify(shardCount, 0, indexStats); } /** * Helper function to test restoring an index having replicas from remote store when all the nodes housing the primary/replica drop. - * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) - throws IOException { + private void testRestoreFlowBothPrimaryReplicasDown(int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { prepareCluster(1, 2, INDEX_NAME, 1, shardCount); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -170,14 +181,7 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int ensureRed(INDEX_NAME); internalCluster().startDataOnlyNodes(2); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); - - ensureGreen(INDEX_NAME); - - assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); - assertEquals(0, getNumShards(INDEX_NAME).numReplicas); - verifyRestoredData(indexStats, true, INDEX_NAME); + restoreAndVerify(shardCount, 1, indexStats); } /** @@ -212,10 +216,16 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices)); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices)); + } client().admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(indices); for (String index : indices) { assertEquals(shardCount, getNumShards(index).totalNumShards); @@ -223,6 +233,37 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo } } + public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException { + int shardCount = randomIntBetween(1, 5); + prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + indexData(randomIntBetween(2, 5), true, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + + PlainActionFuture future = PlainActionFuture.newFuture(); + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(true), future); + try { + future.get(); + } catch (ExecutionException e) { + // If the request goes to co-ordinator, e.getCause() can be RemoteTransportException + assertTrue(e.getCause() instanceof IllegalStateException || e.getCause().getCause() instanceof IllegalStateException); + } + } + + public void testRestoreFlowNoRedIndex() { + int shardCount = randomIntBetween(1, 5); + prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + Map indexStats = indexData(randomIntBetween(2, 5), true, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(false), PlainActionFuture.newFuture()); + + ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + verifyRestoredData(indexStats, true, INDEX_NAME); + } + /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. @@ -265,7 +306,7 @@ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, 1, true, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(1, true, randomIntBetween(1, 5)); } /** @@ -274,7 +315,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOExce */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, 1, false, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(1, false, randomIntBetween(1, 5)); } /** @@ -284,7 +325,7 @@ public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOExc */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } /** @@ -294,7 +335,7 @@ public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOExcepti */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), true, randomIntBetween(1, 5)); } /** @@ -341,10 +382,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices)); - client().admin() - .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(new String[] {}), PlainActionFuture.newFuture()); + restore(indices); ensureGreen(indices); for (String index : indices) { @@ -381,10 +419,16 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + } client().admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indices[0], indices[1]), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(indices[0], indices[1]).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(indices[0], indices[1]); assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); @@ -427,10 +471,16 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + } client().admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*"), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*").restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(indices[0], indices[1]); assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); @@ -490,7 +540,7 @@ private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throw assertEquals(0, recoverySource.get().getIndex().recoveredFileCount()); } - IndexResponse response = indexSingleDoc(); + IndexResponse response = indexSingleDoc(INDEX_NAME); assertEquals(indexStats.get(MAX_SEQ_NO_TOTAL) + 1, response.getSeqNo()); refresh(INDEX_NAME); assertBusy( diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index dbd96a7fd109f..30a836b41e29e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -276,7 +276,10 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); client.admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); ensureYellowAndNoInitializingShards(restoredIndexName1); ensureGreen(restoredIndexName1); assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); @@ -434,7 +437,9 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { // Re-initialize client to make sure we are not using client from stopped node. client = client(clusterManagerNode); assertAcked(client.admin().indices().prepareClose(indexName1)); - client.admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture()); + client.admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(true), PlainActionFuture.newFuture()); ensureYellowAndNoInitializingShards(indexName1); ensureGreen(indexName1); assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); @@ -515,7 +520,10 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); client.admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); ensureYellowAndNoInitializingShards(restoredIndexName1); ensureGreen(restoredIndexName1); // indexing some new docs and validating diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java index 703b9575a88ad..eb1935158c231 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java @@ -35,7 +35,8 @@ public class RestoreRemoteStoreRequest extends ClusterManagerNodeRequest implements ToXContentObject { private String[] indices = Strings.EMPTY_ARRAY; - private Boolean waitForCompletion; + private Boolean waitForCompletion = false; + private Boolean restoreAllShards = false; public RestoreRemoteStoreRequest() {} @@ -43,6 +44,7 @@ public RestoreRemoteStoreRequest(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); waitForCompletion = in.readOptionalBoolean(); + restoreAllShards = in.readOptionalBoolean(); } @Override @@ -50,6 +52,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(indices); out.writeOptionalBoolean(waitForCompletion); + out.writeOptionalBoolean(restoreAllShards); } @Override @@ -118,6 +121,27 @@ public boolean waitForCompletion() { return waitForCompletion; } + /** + * Set the value for restoreAllShards, denoting whether to restore all shards or only unassigned shards + * + * @param restoreAllShards If true, the operation will restore all the shards of the given indices. + * If false, the operation will restore only the unassigned shards of the given indices. + * @return this request + */ + public RestoreRemoteStoreRequest restoreAllShards(boolean restoreAllShards) { + this.restoreAllShards = restoreAllShards; + return this; + } + + /** + * Returns restoreAllShards setting + * + * @return true if the operation will restore all the shards of the given indices + */ + public boolean restoreAllShards() { + return restoreAllShards; + } + /** * Parses restore definition * @@ -167,12 +191,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RestoreRemoteStoreRequest that = (RestoreRemoteStoreRequest) o; - return waitForCompletion == that.waitForCompletion && Arrays.equals(indices, that.indices); + return waitForCompletion == that.waitForCompletion + && restoreAllShards == that.restoreAllShards + && Arrays.equals(indices, that.indices); } @Override public int hashCode() { - int result = Objects.hash(waitForCompletion); + int result = Objects.hash(waitForCompletion, restoreAllShards); result = 31 * result + Arrays.hashCode(indices); return result; } @@ -181,4 +207,5 @@ public int hashCode() { public String toString() { return org.opensearch.common.Strings.toString(XContentType.JSON, this); } + } diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index af348c1c98f2d..781ca5bb2255a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -450,7 +450,11 @@ public Builder initializeAsRestore(IndexMetadata indexMetadata, SnapshotRecovery /** * Initializes an existing index, to be restored from remote store */ - public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, RemoteStoreRecoverySource recoverySource) { + public Builder initializeAsRemoteStoreRestore( + IndexMetadata indexMetadata, + RemoteStoreRecoverySource recoverySource, + Map activeInitializingShards + ) { final UnassignedInfo unassignedInfo = new UnassignedInfo( UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, "restore_source[remote_store]" @@ -462,7 +466,11 @@ public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, Remot for (int shardNumber = 0; shardNumber < indexMetadata.getNumberOfShards(); shardNumber++) { ShardId shardId = new ShardId(index, shardNumber); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo)); + if (activeInitializingShards.containsKey(shardId)) { + indexShardRoutingBuilder.addShard(activeInitializingShards.get(shardId)); + } else { + indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo)); + } shards.put(shardNumber, indexShardRoutingBuilder.build()); } return this; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 1bee5d8176a0f..7934649a6d3eb 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -562,9 +562,13 @@ public Builder addAsFromOpenToClose(IndexMetadata indexMetadata) { return add(indexRoutingBuilder); } - public Builder addAsRemoteStoreRestore(IndexMetadata indexMetadata, RemoteStoreRecoverySource recoverySource) { + public Builder addAsRemoteStoreRestore( + IndexMetadata indexMetadata, + RemoteStoreRecoverySource recoverySource, + Map activeInitializingShards + ) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()) - .initializeAsRemoteStoreRestore(indexMetadata, recoverySource); + .initializeAsRemoteStoreRestore(indexMetadata, recoverySource, activeInitializingShards); add(indexRoutingBuilder); return this; } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java index fca6745167bb4..414c82b4a470f 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java @@ -44,6 +44,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC request.paramAsTime("cluster_manager_timeout", restoreRemoteStoreRequest.masterNodeTimeout()) ); restoreRemoteStoreRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false)); + restoreRemoteStoreRequest.restoreAllShards(request.paramAsBoolean("restore_all_shards", false)); request.applyContentParser(p -> restoreRemoteStoreRequest.source(p.mapOrdered())); return channel -> client.admin().cluster().restoreRemoteStore(restoreRemoteStoreRequest, new RestToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 54d55f67ccdcd..d7e89172c5837 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -62,6 +62,7 @@ import org.opensearch.cluster.metadata.MetadataIndexUpgradeService; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.cluster.routing.RecoverySource.RemoteStoreRecoverySource; @@ -234,21 +235,34 @@ public ClusterState execute(ClusterState currentState) { continue; } if (currentIndexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false)) { - if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) { - throw new IllegalStateException( - "cannot restore index [" - + index - + "] because an open index " - + "with same name already exists in the cluster. Close the existing index" - ); + IndexMetadata updatedIndexMetadata = currentIndexMetadata; + Map activeInitializingShards = new HashMap<>(); + if (request.restoreAllShards()) { + if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) { + throw new IllegalStateException( + "cannot restore index [" + + index + + "] because an open index " + + "with same name already exists in the cluster. Close the existing index" + ); + } + updatedIndexMetadata = IndexMetadata.builder(currentIndexMetadata) + .state(IndexMetadata.State.OPEN) + .version(1 + currentIndexMetadata.getVersion()) + .mappingVersion(1 + currentIndexMetadata.getMappingVersion()) + .settingsVersion(1 + currentIndexMetadata.getSettingsVersion()) + .aliasesVersion(1 + currentIndexMetadata.getAliasesVersion()) + .build(); + } else { + activeInitializingShards = currentState.routingTable() + .index(index) + .shards() + .values() + .stream() + .map(IndexShardRoutingTable::primaryShard) + .filter(shardRouting -> shardRouting.unassigned() == false) + .collect(Collectors.toMap(ShardRouting::shardId, Function.identity())); } - IndexMetadata updatedIndexMetadata = IndexMetadata.builder(currentIndexMetadata) - .state(IndexMetadata.State.OPEN) - .version(1 + currentIndexMetadata.getVersion()) - .mappingVersion(1 + currentIndexMetadata.getMappingVersion()) - .settingsVersion(1 + currentIndexMetadata.getSettingsVersion()) - .aliasesVersion(1 + currentIndexMetadata.getAliasesVersion()) - .build(); IndexId indexId = new IndexId(index, updatedIndexMetadata.getIndexUUID()); @@ -257,7 +271,7 @@ public ClusterState execute(ClusterState currentState) { updatedIndexMetadata.getCreationVersion(), indexId ); - rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource); + rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource, activeInitializingShards); blocks.updateBlocks(updatedIndexMetadata); mdBuilder.put(updatedIndexMetadata, true); indicesToBeRestored.add(index); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java index 81d7074977253..2edfa23286658 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java @@ -38,6 +38,7 @@ private RestoreRemoteStoreRequest randomState(RestoreRemoteStoreRequest instance } instance.waitForCompletion(randomBoolean()); + instance.restoreAllShards(randomBoolean()); if (randomBoolean()) { instance.masterNodeTimeout(randomTimeValue()); @@ -76,6 +77,7 @@ public void testSource() throws IOException { RestoreRemoteStoreRequest processed = new RestoreRemoteStoreRequest(); processed.masterNodeTimeout(original.masterNodeTimeout()); processed.waitForCompletion(original.waitForCompletion()); + processed.restoreAllShards(original.restoreAllShards()); processed.source(map); assertEquals(original, processed); diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java index 6b869ffed7d23..0ff9d6f07751a 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java @@ -49,10 +49,13 @@ import org.junit.Before; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.function.Predicate; +import static org.mockito.Mockito.mock; import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -502,13 +505,40 @@ public void testAddAsRemoteStoreRestore() { Version.CURRENT, new IndexId(TEST_INDEX_1, "1") ); - final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore(indexMetadata, remoteStoreRecoverySource) - .build(); + final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore( + indexMetadata, + remoteStoreRecoverySource, + new HashMap<>() + ).build(); assertTrue(routingTable.hasIndex(TEST_INDEX_1)); assertEquals(this.numberOfShards, routingTable.allShards(TEST_INDEX_1).size()); assertEquals(this.numberOfShards, routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size()); } + public void testAddAsRemoteStoreRestoreWithActiveShards() { + final IndexMetadata indexMetadata = createIndexMetadata(TEST_INDEX_1).state(IndexMetadata.State.OPEN).build(); + final RemoteStoreRecoverySource remoteStoreRecoverySource = new RemoteStoreRecoverySource( + "restore_uuid", + Version.CURRENT, + new IndexId(TEST_INDEX_1, "1") + ); + Map activeInitializingShards = new HashMap<>(); + for (int i = 0; i < randomIntBetween(1, this.numberOfShards); i++) { + activeInitializingShards.put(new ShardId(indexMetadata.getIndex(), i), mock(ShardRouting.class)); + } + final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore( + indexMetadata, + remoteStoreRecoverySource, + activeInitializingShards + ).build(); + assertTrue(routingTable.hasIndex(TEST_INDEX_1)); + assertEquals(this.numberOfShards, routingTable.allShards(TEST_INDEX_1).size()); + assertEquals( + this.numberOfShards - activeInitializingShards.size(), + routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size() + ); + } + /** reverse engineer the in sync aid based on the given indexRoutingTable **/ public static IndexMetadata updateActiveAllocations(IndexRoutingTable indexRoutingTable, IndexMetadata indexMetadata) { IndexMetadata.Builder imdBuilder = IndexMetadata.builder(indexMetadata); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 0c68512f93ea6..cc4fa6f28bafc 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -190,7 +190,6 @@ public void testIsSegmentReplicationAllowed_WrongEngineType() throws IOException * reader close operation on replica shard deletes the segment files copied in current round of segment replication. * It does this by blocking the finalizeReplication on replica shard and performing close operation on acquired * searcher that triggers the reader close operation. - * @throws Exception */ public void testSegmentReplication_With_ReaderClosedConcurrently() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; @@ -240,7 +239,6 @@ public void testSegmentReplication_With_ReaderClosedConcurrently() throws Except /** * Similar to test above, this test shows the issue where an engine close operation during active segment replication * can result in Lucene CorruptIndexException. - * @throws Exception */ public void testSegmentReplication_With_EngineClosedConcurrently() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; @@ -289,7 +287,6 @@ public void testSegmentReplication_With_EngineClosedConcurrently() throws Except /** * Verifies that commits on replica engine resulting from engine or reader close does not cleanup the temporary * replication files from ongoing round of segment replication - * @throws Exception */ public void testTemporaryFilesNotCleanup() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; From 91bc891c1e771dda8742105051731247e4198c73 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 27 Jul 2023 13:30:36 -0400 Subject: [PATCH 169/186] Add 2.9.1 to BWC and known versions (#8923) (#8929) Signed-off-by: Andriy Redko (cherry picked from commit 212dba457d6b2d540694e040a0485529d6e52e12) --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 31b1cb5efe3a1..8cfd636b9fd2e 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -21,4 +21,5 @@ BWC_VERSION: - "2.8.0" - "2.8.1" - "2.9.0" + - "2.9.1" - "2.10.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 9329f221922ea..3f83282245fd8 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -90,6 +90,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_2_9_1 = new Version(2090199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0); public static final Version CURRENT = V_3_0_0; From e2a664c49d5a182a2351293ec56bf9dc9b3311d9 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 27 Jul 2023 16:38:44 -0500 Subject: [PATCH 170/186] Fix flakiness in MasterServiceTests.testThrottlingForMultipleTaskTypes (#8901) * Fix flakiness in MasterServiceTests.testThrottlingForMultipleTaskTypes The test configured a [timeout duration of zero][1] for certain tasks and asserted that all tasks were throttled or timed out. This is not a valid assertion because it is possible for a task to complete before the [asynchronous timeout operation runs][2], which means the task would complete successfully. The fix is to adjust the assertion to allow for successful tasks in this case. [1]: https://github.com/opensearch-project/OpenSearch/blob/60985bc300d9eafd36c1ab25d46235e1c925c565/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java#L941 [2]: https://github.com/opensearch-project/OpenSearch/blob/9fc3f4096958159ec9b53012fc7ced19fd793e1b/server/src/main/java/org/opensearch/common/util/concurrent/PrioritizedOpenSearchThreadPoolExecutor.java#L266 Signed-off-by: Andrew Ross * Add a deterministic test case for timeout Signed-off-by: Andrew Ross --------- Signed-off-by: Andrew Ross --- .../cluster/service/MasterServiceTests.java | 77 ++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java index 3c27748daa87d..d4804b18bd160 100644 --- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java @@ -86,6 +86,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import com.carrotsearch.randomizedtesting.annotations.Timeout; + import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.anyOf; @@ -863,6 +865,7 @@ public ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey( AtomicInteger throttledTask3 = new AtomicInteger(); AtomicInteger succeededTask1 = new AtomicInteger(); AtomicInteger succeededTask2 = new AtomicInteger(); + AtomicInteger succeededTask3 = new AtomicInteger(); AtomicInteger timedOutTask3 = new AtomicInteger(); final ClusterStateTaskListener listener = new ClusterStateTaskListener() { @@ -880,6 +883,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS succeededTask1.incrementAndGet(); } else if (source.equals(task2)) { succeededTask2.incrementAndGet(); + } else if (source.equals(task3)) { + succeededTask3.incrementAndGet(); } latch.countDown(); } @@ -955,7 +960,7 @@ public void run() { assertEquals(numberOfTask1, throttledTask1.get() + succeededTask1.get()); assertEquals(numberOfTask2, succeededTask2.get()); assertEquals(0, throttledTask2.get()); - assertEquals(numberOfTask3, throttledTask3.get() + timedOutTask3.get()); + assertEquals(numberOfTask3, throttledTask3.get() + timedOutTask3.get() + succeededTask3.get()); masterService.close(); } @@ -1378,6 +1383,76 @@ public void testDeprecatedMasterServiceUpdateTaskThreadName() { assertThrows(AssertionError.class, () -> MasterService.assertClusterManagerUpdateThread()); } + @Timeout(millis = 5_000) + public void testTaskTimeout() throws InterruptedException { + try (ClusterManagerService clusterManagerService = createClusterManagerService(true)) { + final AtomicInteger failureCount = new AtomicInteger(); + final AtomicInteger successCount = new AtomicInteger(); + final CountDownLatch taskStartLatch = new CountDownLatch(1); + final CountDownLatch blockingTaskLatch = new CountDownLatch(1); + final CountDownLatch timeoutLatch = new CountDownLatch(1); + final ClusterStateTaskListener blockingListener = new ClusterStateTaskListener() { + @Override + public void onFailure(String source, Exception e) { + fail("Unexpected failure"); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + successCount.incrementAndGet(); + taskStartLatch.countDown(); + try { + blockingTaskLatch.await(); + } catch (InterruptedException e) { + fail("Interrupted"); + } + } + }; + final ClusterStateTaskListener timeoutListener = new ClusterStateTaskListener() { + @Override + public void onFailure(String source, Exception e) { + assertEquals("timeout", source); + failureCount.incrementAndGet(); + timeoutLatch.countDown(); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + fail("Unexpected success"); + } + }; + + final ClusterStateTaskExecutor executor = (currentState, tasks) -> ClusterStateTaskExecutor.ClusterTasksResult.builder() + .successes(tasks) + .build(currentState); + + // start a task and wait for it to start and block on the clusterStateProcessed callback + clusterManagerService.submitStateUpdateTask( + "success", + new Object(), + ClusterStateTaskConfig.build(randomFrom(Priority.values())), + executor, + blockingListener + ); + taskStartLatch.await(); + + // start a second task that is guaranteed to timeout as the first task is still running + clusterManagerService.submitStateUpdateTask( + "timeout", + new Object(), + ClusterStateTaskConfig.build(randomFrom(Priority.values()), TimeValue.timeValueMillis(1L)), + executor, + timeoutListener + ); + + // wait for the timeout to happen, then unblock and assert one success and one failure + timeoutLatch.await(); + blockingTaskLatch.countDown(); + assertEquals(1, failureCount.get()); + assertEquals(1, successCount.get()); + } + } + /** * Returns the cluster state that the cluster-manager service uses (and that is provided by the discovery layer) */ From 4ad418210a51c518119a4c9c565fbf7e9bc4b5c1 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 27 Jul 2023 15:17:15 -0700 Subject: [PATCH 171/186] Fix SegmentReplicationIT.testReplicaHasDiffFilesThanPrimary for node-node replication (#8912) * Fix SegmentReplicationIT.testReplicahasDiffFilesThanPrimary This test is now failing for node-node replication. On the primary shard the prepareSegmentReplication method should cancel any ongoing replication if it is running and start a new sync. Thisis incorrectly using Map.compute which will not replace the existing handler entry in the allocationIdToHandlers map. It will only cancel the existing source handler. As a result this can leave the copyState map with an entry and hold an index commit while the test is cleaning up. The copyState is only cleared when a handler is cancelled directly or from a cluster state update. Signed-off-by: Marc Handalian * PR feedback. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../OngoingSegmentReplications.java | 26 ++++++++++---- .../OngoingSegmentReplicationsTests.java | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java index e0e356f1531e1..4712ae6c18759 100644 --- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java +++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java @@ -139,13 +139,25 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener { - if (segrepHandler != null) { - logger.warn("Override handler for allocation id {}", request.getTargetAllocationId()); - cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry"); - } - return createTargetHandler(request.getTargetNode(), copyState, request.getTargetAllocationId(), fileChunkWriter); - }); + final SegmentReplicationSourceHandler newHandler = createTargetHandler( + request.getTargetNode(), + copyState, + request.getTargetAllocationId(), + fileChunkWriter + ); + final SegmentReplicationSourceHandler existingHandler = allocationIdToHandlers.putIfAbsent( + request.getTargetAllocationId(), + newHandler + ); + // If we are already replicating to this allocation Id, cancel the old and replace with a new execution. + // This will clear the old handler & referenced copy state holding an incref'd indexCommit. + if (existingHandler != null) { + logger.warn("Override handler for allocation id {}", request.getTargetAllocationId()); + cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry"); + assert allocationIdToHandlers.containsKey(request.getTargetAllocationId()) == false; + allocationIdToHandlers.put(request.getTargetAllocationId(), newHandler); + } + assert allocationIdToHandlers.containsKey(request.getTargetAllocationId()); return copyState; } diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java index 3b289114f5ca1..84a53ae22a6bc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java @@ -403,4 +403,38 @@ public void testCancelForMissingIds() throws IOException { assertEquals(0, replications.cachedCopyStateSize()); closeShards(replica_2); } + + public void testPrepareForReplicationAlreadyReplicating() throws IOException { + OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings); + final String replicaAllocationId = replica.routingEntry().allocationId().getId(); + final CheckpointInfoRequest request = new CheckpointInfoRequest(1L, replicaAllocationId, primaryDiscoveryNode, testCheckpoint); + + final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class)); + + final SegmentReplicationSourceHandler handler = replications.getHandlers().get(replicaAllocationId); + assertEquals(handler.getCopyState(), copyState); + assertEquals(1, copyState.refCount()); + + ReplicationCheckpoint secondCheckpoint = new ReplicationCheckpoint( + testCheckpoint.getShardId(), + testCheckpoint.getPrimaryTerm(), + testCheckpoint.getSegmentsGen(), + testCheckpoint.getSegmentInfosVersion() + 1, + testCheckpoint.getCodec() + ); + + final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest( + 1L, + replicaAllocationId, + primaryDiscoveryNode, + secondCheckpoint + ); + + final CopyState secondCopyState = replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class)); + final SegmentReplicationSourceHandler secondHandler = replications.getHandlers().get(replicaAllocationId); + assertEquals(secondHandler.getCopyState(), secondCopyState); + assertEquals("New copy state is incref'd", 1, secondCopyState.refCount()); + assertEquals("Old copy state is cleaned up", 0, copyState.refCount()); + + } } From 5495c641f894bc79aec5bbe4767b3fc7710ecbe0 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 27 Jul 2023 17:56:38 -0500 Subject: [PATCH 172/186] Configure test-retry plugin filter properly (#8933) The intent of #8825 was to retry only specified tests. The wrong parameter was configured though: ['filter' should be set][1], not 'classRetry'. [1]: https://github.com/gradle/test-retry-gradle-plugin/blob/main/README.adoc#filtering Signed-off-by: Andrew Ross --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c7b6987b1103f..12499bbf6a817 100644 --- a/build.gradle +++ b/build.gradle @@ -470,7 +470,7 @@ subprojects { maxFailures = 10 } failOnPassedAfterRetry = false - classRetry { + filter { includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests") includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT") includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT") From 3c973bafe709565380eb4224242e860499fb4acc Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Thu, 27 Jul 2023 18:56:42 -0500 Subject: [PATCH 173/186] [Refactor] MediaTypeParserRegistry to MediaTypeRegistry (#8940) This commit rote refactors MediaTypeParserRegistry to MediaTypeRegistry to make the class naming align with the intention of the logic. The MediaTypeRegistry is a mechanism for downstream extensions to register concrete MediaTypes thus having Parser in the name is unneeded. Signed-off-by: Nicholas Walter Knize --- .../client/indices/CreateIndexRequest.java | 4 +-- .../client/indices/PutMappingRequest.java | 4 +-- .../core/common/io/stream/StreamInput.java | 4 +-- .../opensearch/core/xcontent/MediaType.java | 4 +-- ...erRegistry.java => MediaTypeRegistry.java} | 2 +- .../common/xcontent/XContentType.java | 4 +-- .../common/xcontent/MediaTypeParserTests.java | 27 +++++++++---------- .../opensearch/rest/AbstractRestChannel.java | 4 +-- .../transport/TransportService.java | 4 +-- 9 files changed, 27 insertions(+), 30 deletions(-) rename libs/core/src/main/java/org/opensearch/core/xcontent/{MediaTypeParserRegistry.java => MediaTypeRegistry.java} (99%) diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java index 3405e7e81e122..16915b32c16fe 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java @@ -47,7 +47,7 @@ import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -187,7 +187,7 @@ public CreateIndexRequest mapping(XContentBuilder source) { */ public CreateIndexRequest mapping(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeParserRegistry.getDefaultMediaType()); + XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); builder.map(source); return mapping(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java index d17dc54713789..721d6094f7502 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java @@ -40,7 +40,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -111,7 +111,7 @@ public MediaType mediaType() { */ public PutMappingRequest source(Map mappingSource) { try { - XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeParserRegistry.getDefaultMediaType()); + XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); builder.map(mappingSource); return source(builder); } catch (IOException e) { diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java index 1d7321bf2c6de..d9040da569345 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java @@ -54,7 +54,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import java.io.ByteArrayInputStream; import java.io.EOFException; @@ -347,7 +347,7 @@ public BigInteger readBigInteger() throws IOException { } public MediaType readMediaType() throws IOException { - return MediaTypeParserRegistry.fromMediaType(readString()); + return MediaTypeRegistry.fromMediaType(readString()); } @Nullable diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java index c1409e551e47d..7193cd3bd97bb 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java @@ -82,7 +82,7 @@ default String mediaType() { * This method will return {@code null} if no match is found */ static MediaType fromFormat(String mediaType) { - return MediaTypeParserRegistry.fromFormat(mediaType); + return MediaTypeRegistry.fromFormat(mediaType); } /** @@ -93,7 +93,7 @@ static MediaType fromFormat(String mediaType) { */ static MediaType fromMediaType(String mediaTypeHeaderValue) { mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue); - return MediaTypeParserRegistry.fromMediaType(mediaTypeHeaderValue); + return MediaTypeRegistry.fromMediaType(mediaTypeHeaderValue); } /** diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParserRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParserRegistry.java rename to libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java index 62a26b4458b09..8ac92504a12d8 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParserRegistry.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java @@ -41,7 +41,7 @@ * * @opensearch.internal */ -public final class MediaTypeParserRegistry { +public final class MediaTypeRegistry { private static Map formatToMediaType = Map.of(); private static Map typeWithSubtypeToMediaType = Map.of(); diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java index 023caa49e1f39..9291981f32113 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java @@ -38,7 +38,7 @@ import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContent; import java.io.IOException; @@ -133,7 +133,7 @@ public XContent xContent() { static { /** a parser of media types */ - MediaTypeParserRegistry.register(XContentType.values(), Map.of("application/*", JSON, "application/x-ndjson", JSON)); + MediaTypeRegistry.register(XContentType.values(), Map.of("application/*", JSON, "application/x-ndjson", JSON)); } private int index; diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java index 15492b7351984..64d36f0a8b78f 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.xcontent; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.test.OpenSearchTestCase; import java.util.Collections; @@ -46,40 +46,37 @@ public class MediaTypeParserTests extends OpenSearchTestCase { public void testJsonWithParameters() throws Exception { String mediaType = "application/json"; - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap())); - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8"))); assertThat( - MediaTypeParserRegistry.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), - equalTo(Map.of("charset", "utf-8")) - ); - assertThat( - MediaTypeParserRegistry.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(), + MediaTypeRegistry.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); } public void testWhiteSpaceInTypeSubtype() { String mediaType = " application/json "; - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON)); assertThat( - MediaTypeParserRegistry.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(), + MediaTypeRegistry.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); assertThat( - MediaTypeParserRegistry.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(), + MediaTypeRegistry.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); mediaType = " application / json "; - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType), is(nullValue())); } public void testInvalidParameters() { String mediaType = "application/json"; - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue())); - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + "; key = value"), is(nullValue())); - assertThat(MediaTypeParserRegistry.parseMediaType(mediaType + "; key="), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; key = value"), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; key="), is(nullValue())); } } diff --git a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java index dcee6500325b9..32499b1fc155b 100644 --- a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java @@ -36,7 +36,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -132,7 +132,7 @@ public XContentBuilder newBuilder(@Nullable MediaType requestContentType, @Nulla responseContentType = requestContentType; } else { // default to JSON output when all else fails - responseContentType = MediaTypeParserRegistry.getDefaultMediaType(); + responseContentType = MediaTypeRegistry.getDefaultMediaType(); } } diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index b8d7d130e846b..c3e287b458fc5 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -62,7 +62,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.core.xcontent.MediaTypeParserRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.node.NodeClosedException; import org.opensearch.node.ReportingService; import org.opensearch.tasks.Task; @@ -174,7 +174,7 @@ public void close() {} /** Registers OpenSearch server specific exceptions (exceptions outside of core library) */ OpenSearchServerException.registerExceptions(); // set the default media type to JSON (fallback if a media type is not specified) - MediaTypeParserRegistry.setDefaultMediaType(XContentType.JSON); + MediaTypeRegistry.setDefaultMediaType(XContentType.JSON); } /** does nothing. easy way to ensure class is loaded so the above static block is called to register the streamables */ From 8edc1ddd030e6b867db6f3029cca32a684154632 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Thu, 27 Jul 2023 19:00:31 -0700 Subject: [PATCH 174/186] [Segment Replication] Use deterministic mechanism to have concurrent invocation of segment replication (#8937) * [Segment Replication] Use deterministic mechanism to have concurrent invocation of segment replication Signed-off-by: Suraj Singh * Clean up Signed-off-by: Suraj Singh --------- Signed-off-by: Suraj Singh --- .../SegmentReplicationTargetServiceTests.java | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 4643615d45d7e..94e57f4a0d3e4 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -32,6 +32,7 @@ import org.opensearch.indices.recovery.ForceSyncRequest; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.common.CopyState; import org.opensearch.indices.replication.common.ReplicationCollection; import org.opensearch.indices.replication.common.ReplicationFailedException; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; @@ -49,6 +50,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.atLeastOnce; @@ -70,10 +72,7 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { private IndexShard replicaShard; private IndexShard primaryShard; private ReplicationCheckpoint checkpoint; - private SegmentReplicationSource replicationSource; private SegmentReplicationTargetService sut; - - private ReplicationCheckpoint initialCheckpoint; private ReplicationCheckpoint aheadCheckpoint; private ReplicationCheckpoint newPrimaryCheckpoint; @@ -83,11 +82,10 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { private DiscoveryNode localNode; private IndicesService indicesService; - private ClusterService clusterService; private SegmentReplicationState state; - private static long TRANSPORT_TIMEOUT = 30000;// 30sec + private static final long TRANSPORT_TIMEOUT = 30000;// 30sec @Override public void setUp() throws Exception { @@ -107,9 +105,6 @@ public void setUp() throws Exception { 0L, replicaShard.getLatestReplicationCheckpoint().getCodec() ); - SegmentReplicationSourceFactory replicationSourceFactory = mock(SegmentReplicationSourceFactory.class); - replicationSource = mock(SegmentReplicationSource.class); - when(replicationSourceFactory.get(replicaShard)).thenReturn(replicationSource); testThreadPool = new TestThreadPool("test", Settings.EMPTY); localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); @@ -126,7 +121,7 @@ public void setUp() throws Exception { transportService.acceptIncomingRequests(); indicesService = mock(IndicesService.class); - clusterService = mock(ClusterService.class); + ClusterService clusterService = mock(ClusterService.class); ClusterState clusterState = mock(ClusterState.class); RoutingTable mockRoutingTable = mock(RoutingTable.class); when(clusterService.state()).thenReturn(clusterState); @@ -135,7 +130,7 @@ public void setUp() throws Exception { when(clusterState.nodes()).thenReturn(DiscoveryNodes.builder().add(localNode).build()); sut = prepareForReplication(primaryShard, replicaShard, transportService, indicesService, clusterService); - initialCheckpoint = replicaShard.getLatestReplicationCheckpoint(); + ReplicationCheckpoint initialCheckpoint = replicaShard.getLatestReplicationCheckpoint(); aheadCheckpoint = new ReplicationCheckpoint( initialCheckpoint.getShardId(), initialCheckpoint.getPrimaryTerm(), @@ -242,7 +237,46 @@ public void testAlreadyOnNewCheckpoint() { } public void testShardAlreadyReplicating() { - sut.startReplication(replicaShard, mock(SegmentReplicationTargetService.SegmentReplicationListener.class)); + CountDownLatch blockGetCheckpointMetadata = new CountDownLatch(1); + SegmentReplicationSource source = new TestReplicationSource() { + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + try { + blockGetCheckpointMetadata.await(); + final CopyState copyState = new CopyState( + ReplicationCheckpoint.empty(primaryShard.shardId(), primaryShard.getLatestReplicationCheckpoint().getCodec()), + primaryShard + ); + listener.onResponse( + new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes()) + ); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + ActionListener listener + ) { + listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); + } + }; + final SegmentReplicationTarget target = spy( + new SegmentReplicationTarget(replicaShard, source, mock(SegmentReplicationTargetService.SegmentReplicationListener.class)) + ); + // Start first round of segment replication. + sut.startReplication(target); + + // Start second round of segment replication, this should fail to start as first round is still in-progress sut.startReplication(replicaShard, new SegmentReplicationTargetService.SegmentReplicationListener() { @Override public void onReplicationDone(SegmentReplicationState state) { @@ -255,6 +289,7 @@ public void onReplicationFailure(SegmentReplicationState state, ReplicationFaile assertFalse(sendShardFailure); } }); + blockGetCheckpointMetadata.countDown(); } public void testOnNewCheckpointFromNewPrimaryCancelOngoingReplication() throws InterruptedException { From 3952d5e0be809d1fa42d37618edf5f31448e7053 Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Thu, 27 Jul 2023 21:15:01 -0500 Subject: [PATCH 175/186] Register MediaTypes through SPI (#8938) * Register MediaTypes through SPI This commit provides a new SPI interface MediaContentProvider. Modules, Plugins, Extensions, implement this interface and provide the concrete MediaType implementations (and MIME aliases) through getMediaTypes and getAdditionalMediaTypes, respectively. This enables downstream extensions (e.g., serverless or cloud native implementations) to register their own custom MediaType and define the serialization format that is registered when the classloader loads the MediaTypeRegistry instead of having to register the types explicitly in application code. Signed-off-by: Nicholas Walter Knize * pass the MediaTypeProvider classloader to the SPI ServiceLoader Signed-off-by: Nicholas Walter Knize --------- Signed-off-by: Nicholas Walter Knize --- .../core/xcontent/MediaTypeRegistry.java | 29 +++++++++++++-- .../core/xcontent/spi/MediaTypeProvider.java | 29 +++++++++++++++ .../core/xcontent/spi/package-info.java | 10 ++++++ .../common/xcontent/XContentType.java | 7 ---- .../common/xcontent/spi/XContentProvider.java | 35 +++++++++++++++++++ .../common/xcontent/spi/package-info.java | 10 ++++++ ...search.core.xcontent.spi.MediaTypeProvider | 9 +++++ .../transport/TransportService.java | 4 --- 8 files changed, 120 insertions(+), 13 deletions(-) create mode 100644 libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java create mode 100644 libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java create mode 100644 libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java create mode 100644 libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java create mode 100644 libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java index 8ac92504a12d8..b81325f6c7c74 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java @@ -32,9 +32,16 @@ package org.opensearch.core.xcontent; +import org.opensearch.core.xcontent.spi.MediaTypeProvider; + +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Parses supported internet media types @@ -48,7 +55,25 @@ public final class MediaTypeRegistry { // Default mediaType singleton private static MediaType DEFAULT_MEDIA_TYPE; - public static void register(MediaType[] acceptedMediaTypes, Map additionalMediaTypes) { + // JSON is a core type, so we create a static instance for implementations that require JSON format (e.g., tests) + // todo we should explore moving the concrete JSON implementation from the xcontent library to core + public static final MediaType JSON; + + static { + List mediaTypes = new ArrayList<>(); + Map amt = new HashMap<>(); + for (MediaTypeProvider provider : ServiceLoader.load(MediaTypeProvider.class, MediaTypeProvider.class.getClassLoader())) { + mediaTypes.addAll(provider.getMediaTypes()); + amt = Stream.of(amt, provider.getAdditionalMediaTypes()) + .flatMap(map -> map.entrySet().stream()) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + } + register(mediaTypes.toArray(new MediaType[0]), amt); + JSON = fromMediaType("application/json"); + setDefaultMediaType(JSON); + } + + private static void register(MediaType[] acceptedMediaTypes, Map additionalMediaTypes) { // ensures the map is not overwritten: Map typeMap = new HashMap<>(typeWithSubtypeToMediaType); Map formatMap = new HashMap<>(formatToMediaType); @@ -150,7 +175,7 @@ public Map getParameters() { } } - public static void setDefaultMediaType(final MediaType mediaType) { + private static void setDefaultMediaType(final MediaType mediaType) { if (DEFAULT_MEDIA_TYPE != null) { throw new RuntimeException( "unable to reset the default media type from current default [" + DEFAULT_MEDIA_TYPE + "] to [" + mediaType + "]" diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java new file mode 100644 index 0000000000000..eeaadc1698df6 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core.xcontent.spi; + +import org.opensearch.core.xcontent.MediaType; + +import java.util.List; +import java.util.Map; + +/** + * Service Provider Interface for plugins, modules, extensions providing + * their own Media Types + * + * @opensearch.experimental + * @opensearch.api + */ +public interface MediaTypeProvider { + /** Extensions that implement their own concrete {@link MediaType}s provide them through this interface method */ + List getMediaTypes(); + + /** Extensions that implement additional {@link MediaType} aliases provide them through this interface method */ + Map getAdditionalMediaTypes(); +} diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java new file mode 100644 index 0000000000000..67ccd981dafa8 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Service Provider Interface for extensible media types */ +package org.opensearch.core.xcontent.spi; diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java index 9291981f32113..d418e25ba8292 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java @@ -38,11 +38,9 @@ import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContent; import java.io.IOException; -import java.util.Map; /** * The content type of {@link XContent}. @@ -131,11 +129,6 @@ public XContent xContent() { } }; - static { - /** a parser of media types */ - MediaTypeRegistry.register(XContentType.values(), Map.of("application/*", JSON, "application/x-ndjson", JSON)); - } - private int index; XContentType(int index) { diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java new file mode 100644 index 0000000000000..af5ab67507b81 --- /dev/null +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.xcontent.spi; + +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.spi.MediaTypeProvider; + +import java.util.List; +import java.util.Map; + +/** + * Media Type implementations provided by xcontent library + * + * @opensearch.internal + */ +public class XContentProvider implements MediaTypeProvider { + /** Returns the concrete {@link MediaType} provided by the xcontent library */ + @Override + public List getMediaTypes() { + return List.of(XContentType.values()); + } + + /** Returns the additional {@link MediaType} aliases provided by the xcontent library */ + @Override + public Map getAdditionalMediaTypes() { + return Map.of("application/*", XContentType.JSON, "application/x-ndjson", XContentType.JSON); + } +} diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java new file mode 100644 index 0000000000000..c265021f12763 --- /dev/null +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** SPI implementation for the xcontent library */ +package org.opensearch.common.xcontent.spi; diff --git a/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider b/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider new file mode 100644 index 0000000000000..ce3fab93087dd --- /dev/null +++ b/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider @@ -0,0 +1,9 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +org.opensearch.common.xcontent.spi.XContentProvider diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index c3e287b458fc5..25293bf97b222 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -59,10 +59,8 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.node.NodeClosedException; import org.opensearch.node.ReportingService; import org.opensearch.tasks.Task; @@ -173,8 +171,6 @@ public void close() {} Streamables.registerStreamables(); /** Registers OpenSearch server specific exceptions (exceptions outside of core library) */ OpenSearchServerException.registerExceptions(); - // set the default media type to JSON (fallback if a media type is not specified) - MediaTypeRegistry.setDefaultMediaType(XContentType.JSON); } /** does nothing. easy way to ensure class is loaded so the above static block is called to register the streamables */ From 723cab632af71e197acee25290def8d6d4b46343 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Thu, 27 Jul 2023 22:39:58 -0700 Subject: [PATCH 176/186] Remove testUpdateSegmentsWhileRefreshing flaky test failure (#8943) Signed-off-by: Suraj Singh --- .../NRTReplicationReaderManagerTests.java | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java index 98f1a416731e4..d635b38e811c4 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java @@ -46,43 +46,4 @@ public void testCreateNRTreaderManager() throws IOException { } } } - - public void testUpdateSegmentsWhileRefreshing() throws IOException, InterruptedException { - try (final Store store = createStore()) { - store.createEmpty(Version.LATEST); - final DirectoryReader reader = DirectoryReader.open(store.directory()); - NRTReplicationReaderManager readerManager = new NRTReplicationReaderManager( - OpenSearchDirectoryReader.wrap(reader, shardId), - (files) -> {}, - (files) -> {} - ); - - final SegmentInfos infos_2 = readerManager.getSegmentInfos().clone(); - infos_2.changed(); - - Thread refreshThread = new Thread(() -> { - try { - readerManager.maybeRefresh(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - Thread updateThread = new Thread(() -> { - try { - readerManager.updateSegments(infos_2); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - refreshThread.start(); - updateThread.start(); - refreshThread.join(); - updateThread.join(); - try (final OpenSearchDirectoryReader acquire = readerManager.acquire()) { - final StandardDirectoryReader standardReader = NRTReplicationReaderManager.unwrapStandardReader(acquire); - assertEquals(infos_2.version, standardReader.getSegmentInfos().version); - } - assertEquals(infos_2, readerManager.getSegmentInfos()); - } - } } From c43743dd8ac132314dfe375f712c37a7a0f7d8df Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 28 Jul 2023 12:41:46 +0530 Subject: [PATCH 177/186] amending tests to include all codecs (#8907) Signed-off-by: Sarthak Aggarwal --- .../test/java/org/opensearch/upgrades/IndexingIT.java | 4 ++-- .../indices/replication/SegmentReplicationIT.java | 6 +----- .../org/opensearch/test/OpenSearchIntegTestCase.java | 10 +++++++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index b60ee09d39048..d196ec3dcd0a8 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -40,7 +40,6 @@ import org.opensearch.common.Booleans; import org.opensearch.common.io.Streams; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.EngineConfig; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.rest.yaml.ObjectPath; @@ -54,6 +53,7 @@ import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; import static org.opensearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; +import static org.opensearch.test.OpenSearchIntegTestCase.CODECS; /** * Basic test that indexed documents survive the rolling restart. See @@ -267,7 +267,7 @@ public void testIndexingWithSegRep() throws Exception { .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put( EngineConfig.INDEX_CODEC_SETTING.getKey(), - randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) + randomFrom(CODECS) ) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms"); createIndex(indexName, settings.build()); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 3ab1a2a8564c5..405d0b595e02b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -53,7 +53,6 @@ import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; -import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.NRTReplicationReaderManager; @@ -201,10 +200,7 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { final String nodeB = internalCluster().startDataOnlyNode(); final Settings settings = Settings.builder() .put(indexSettings()) - .put( - EngineConfig.INDEX_CODEC_SETTING.getKey(), - randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) - ) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CODECS)) .build(); createIndex(INDEX_NAME, settings); ensureGreen(INDEX_NAME); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 3564bd667ee2b..6e3364b208180 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -364,6 +364,14 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private static OpenSearchIntegTestCase INSTANCE = null; // see @SuiteScope private static Long SUITE_SEED = null; + public static List CODECS = List.of( + CodecService.DEFAULT_CODEC, + CodecService.BEST_COMPRESSION_CODEC, + CodecService.LUCENE_DEFAULT_CODEC, + CodecService.ZSTD_CODEC, + CodecService.ZSTD_NO_DICT_CODEC + ); + @BeforeClass public static void beforeClass() throws Exception { SUITE_SEED = randomLong(); @@ -427,7 +435,7 @@ protected void randomIndexTemplate() { // otherwise, use it, it has assertions and so on that can find bugs. SuppressCodecs annotation = getClass().getAnnotation(SuppressCodecs.class); if (annotation != null && annotation.value().length == 1 && "*".equals(annotation.value()[0])) { - randomSettingsBuilder.put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)); + randomSettingsBuilder.put("index.codec", randomFrom(CODECS)); } else { randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC); } From 96630f036f80708126ef4a2459577dab629220ed Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:53:09 +0530 Subject: [PATCH 178/186] Add metadata prefix to Remote Translog Metadata file (#8914) Signed-off-by: Gaurav Bafna --- .../transfer/BlobStoreTransferService.java | 7 ++-- .../translog/transfer/TransferService.java | 12 +++++-- .../transfer/TranslogTransferManager.java | 8 ++++- .../transfer/TranslogTransferMetadata.java | 3 ++ .../TranslogTransferManagerTests.java | 32 +++++++++++++------ .../blobstore/BlobStoreRepositoryTests.java | 2 -- 6 files changed, 45 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index 974e8af42b939..95424d86fba34 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -213,17 +213,18 @@ public void listFoldersAsync(String threadpoolName, Iterable path, Actio }); } - public void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener) { - blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder("", limit, LEXICOGRAPHIC, listener); + public void listAllInSortedOrder(Iterable path, String filenamePrefix, int limit, ActionListener> listener) { + blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder(filenamePrefix, limit, LEXICOGRAPHIC, listener); } public void listAllInSortedOrderAsync( String threadpoolName, Iterable path, + String filenamePrefix, int limit, ActionListener> listener ) { - threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, limit, listener); }); + threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, filenamePrefix, limit, listener); }); } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java index a240fd38cda11..885a2e32e915d 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java @@ -125,8 +125,14 @@ void uploadBlobs( */ InputStream downloadBlob(Iterable path, String fileName) throws IOException; - void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener); - - void listAllInSortedOrderAsync(String threadpoolName, Iterable path, int limit, ActionListener> listener); + void listAllInSortedOrder(Iterable path, String filenamePrefix, int limit, ActionListener> listener); + + void listAllInSortedOrderAsync( + String threadpoolName, + Iterable path, + String filenamePrefix, + int limit, + ActionListener> listener + ); } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index e2bb5f74df234..850a0c1a6574c 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -210,7 +210,12 @@ public TranslogTransferMetadata readMetadata() throws IOException { ); try { - transferService.listAllInSortedOrder(remoteMetadataTransferPath, 1, latchedActionListener); + transferService.listAllInSortedOrder( + remoteMetadataTransferPath, + TranslogTransferMetadata.METADATA_PREFIX, + 1, + latchedActionListener + ); latch.await(); } catch (InterruptedException e) { throw new IOException("Exception while reading/downloading metadafile", e); @@ -367,6 +372,7 @@ public void deleteStaleTranslogMetadataFilesAsync(Runnable onCompletion) { transferService.listAllInSortedOrderAsync( ThreadPool.Names.REMOTE_PURGE, remoteMetadataTransferPath, + TranslogTransferMetadata.METADATA_PREFIX, Integer.MAX_VALUE, new ActionListener<>() { @Override diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index 75d6549b23f1e..a8b3404d3f2ce 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -36,6 +36,8 @@ public class TranslogTransferMetadata { public static final String METADATA_SEPARATOR = "__"; + public static final String METADATA_PREFIX = "metadata"; + static final int BUFFER_SIZE = 4096; static final int CURRENT_VERSION = 1; @@ -83,6 +85,7 @@ public String getFileName() { return String.join( METADATA_SEPARATOR, Arrays.asList( + METADATA_PREFIX, RemoteStoreUtils.invertLong(primaryTerm), RemoteStoreUtils.invertLong(generation), RemoteStoreUtils.invertLong(createdAt), diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index b7091f3f4f8a6..3d622d6bdf8b8 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -44,6 +44,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -205,11 +206,12 @@ public void testReadMetadataNoFile() throws IOException { null ); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); latchedActionListener.onResponse(bmList); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); assertNull(translogTransferManager.readMetadata()); } @@ -225,12 +227,13 @@ public void testReadMetadataSingleFile() throws IOException { TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2); String mdFilename = tm.getFileName(); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(mdFilename, 1)); latchedActionListener.onResponse(bmList); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata(); when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenReturn( @@ -252,12 +255,13 @@ public void testReadMetadataReadException() throws IOException { String mdFilename = tm.getFileName(); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(mdFilename, 1)); latchedActionListener.onResponse(bmList); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenThrow(new IOException("Something went wrong")); @@ -283,10 +287,11 @@ public void testReadMetadataListException() throws IOException { ); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); latchedActionListener.onFailure(new IOException("Issue while listing")); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); when(transferService.downloadBlob(any(BlobPath.class), any(String.class))).thenThrow(new IOException("Something went wrong")); @@ -416,7 +421,7 @@ public void testDeleteStaleTranslogMetadata() { String tm2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName(); String tm3 = new TranslogTransferMetadata(2, 3, 1, 2).getFileName(); doAnswer(invocation -> { - ActionListener> actionListener = invocation.getArgument(3); + ActionListener> actionListener = invocation.getArgument(4); List bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(tm1, 1)); bmList.add(new PlainBlobMetadata(tm2, 1)); @@ -424,12 +429,19 @@ public void testDeleteStaleTranslogMetadata() { actionListener.onResponse(bmList); return null; }).when(transferService) - .listAllInSortedOrderAsync(eq(ThreadPool.Names.REMOTE_PURGE), any(BlobPath.class), anyInt(), any(ActionListener.class)); + .listAllInSortedOrderAsync( + eq(ThreadPool.Names.REMOTE_PURGE), + any(BlobPath.class), + anyString(), + anyInt(), + any(ActionListener.class) + ); List files = List.of(tm2, tm3); translogTransferManager.deleteStaleTranslogMetadataFilesAsync(() -> { verify(transferService).listAllInSortedOrderAsync( eq(ThreadPool.Names.REMOTE_PURGE), any(BlobPath.class), + eq(TranslogTransferMetadata.METADATA_PREFIX), eq(Integer.MAX_VALUE), any() ); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 28513f279f8ad..9d711d464754c 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -32,7 +32,6 @@ package org.opensearch.repositories.blobstore; -import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.Version; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -91,7 +90,6 @@ /** * Tests for the {@link BlobStoreRepository} and its subclasses. */ -@LuceneTestCase.SuppressFileSystems("ExtrasFS") public class BlobStoreRepositoryTests extends OpenSearchSingleNodeTestCase { static final String REPO_TYPE = "fsLike"; From 106e83a75205e5ff5a3a760b28a53f71240e7e61 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 28 Jul 2023 10:27:36 -0400 Subject: [PATCH 179/186] [BUG] CompletionSuggestSearchIT.testSkipDuplicates is flaky (#8963) Signed-off-by: Andriy Redko --- .../search/internal/ContextIndexSearcher.java | 4 ++++ .../suggest/completion/CompletionSuggester.java | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index e3ca932eb4699..5384b47cc69ec 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -343,6 +343,10 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } } } + + // Note: this is called if collection ran successfully, including the above special cases of + // CollectionTerminatedException and TimeExceededException, but no other exception. + leafCollector.finish(); } private Weight wrapWeight(Weight weight) { diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java index 175503cb94e3d..df18e874be9c9 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java @@ -35,6 +35,7 @@ import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.TopSuggestDocs; @@ -108,15 +109,21 @@ private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSu for (LeafReaderContext context : searcher.getIndexReader().leaves()) { BulkScorer scorer = weight.bulkScorer(context); if (scorer != null) { + LeafCollector leafCollector = null; try { - scorer.score(collector.getLeafCollector(context), context.reader().getLiveDocs()); + leafCollector = collector.getLeafCollector(context); + scorer.score(leafCollector, context.reader().getLiveDocs()); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf } + // Note: this is called if collection ran successfully, including the above special cases of + // CollectionTerminatedException and TimeExceededException, but no other exception. + if (leafCollector != null) { + leafCollector.finish(); + } } } - collector.finish(); } @Override From 0003bd86332b8a59a3581bc4f48bdb3bf69ecab0 Mon Sep 17 00:00:00 2001 From: Evan Kielley Date: Fri, 28 Jul 2023 07:36:56 -0700 Subject: [PATCH 180/186] Added empty build.gradle files for arm64 and s390x because the gradle assemble task fails without them. (#8904) Signed-off-by: Evan Kielley --- .../docker/docker-arm64-export/build.gradle | 13 +++++++++++++ .../docker/docker-s390x-export/build.gradle | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 distribution/docker/docker-arm64-export/build.gradle create mode 100644 distribution/docker/docker-s390x-export/build.gradle diff --git a/distribution/docker/docker-arm64-export/build.gradle b/distribution/docker/docker-arm64-export/build.gradle new file mode 100644 index 0000000000000..3506c4e39c234 --- /dev/null +++ b/distribution/docker/docker-arm64-export/build.gradle @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/docker/docker-s390x-export/build.gradle b/distribution/docker/docker-s390x-export/build.gradle new file mode 100644 index 0000000000000..3506c4e39c234 --- /dev/null +++ b/distribution/docker/docker-s390x-export/build.gradle @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +// This file is intentionally blank. All configuration of the +// export is done in the parent project. From b4b25dc0ef845d0f7a45b7db0ab931dcf783c560 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 28 Jul 2023 12:32:16 -0400 Subject: [PATCH 181/186] Revert "amending tests to include all codecs (#8907)" (#8968) This reverts commit c43743dd8ac132314dfe375f712c37a7a0f7d8df. Signed-off-by: Andriy Redko --- .../test/java/org/opensearch/upgrades/IndexingIT.java | 4 ++-- .../indices/replication/SegmentReplicationIT.java | 6 +++++- .../org/opensearch/test/OpenSearchIntegTestCase.java | 10 +--------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index d196ec3dcd0a8..b60ee09d39048 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -40,6 +40,7 @@ import org.opensearch.common.Booleans; import org.opensearch.common.io.Streams; import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.EngineConfig; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.rest.yaml.ObjectPath; @@ -53,7 +54,6 @@ import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; import static org.opensearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.opensearch.test.OpenSearchIntegTestCase.CODECS; /** * Basic test that indexed documents survive the rolling restart. See @@ -267,7 +267,7 @@ public void testIndexingWithSegRep() throws Exception { .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put( EngineConfig.INDEX_CODEC_SETTING.getKey(), - randomFrom(CODECS) + randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) ) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms"); createIndex(indexName, settings.build()); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 405d0b595e02b..3ab1a2a8564c5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -53,6 +53,7 @@ import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.NRTReplicationReaderManager; @@ -200,7 +201,10 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { final String nodeB = internalCluster().startDataOnlyNode(); final Settings settings = Settings.builder() .put(indexSettings()) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CODECS)) + .put( + EngineConfig.INDEX_CODEC_SETTING.getKey(), + randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) + ) .build(); createIndex(INDEX_NAME, settings); ensureGreen(INDEX_NAME); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 6e3364b208180..3564bd667ee2b 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -364,14 +364,6 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private static OpenSearchIntegTestCase INSTANCE = null; // see @SuiteScope private static Long SUITE_SEED = null; - public static List CODECS = List.of( - CodecService.DEFAULT_CODEC, - CodecService.BEST_COMPRESSION_CODEC, - CodecService.LUCENE_DEFAULT_CODEC, - CodecService.ZSTD_CODEC, - CodecService.ZSTD_NO_DICT_CODEC - ); - @BeforeClass public static void beforeClass() throws Exception { SUITE_SEED = randomLong(); @@ -435,7 +427,7 @@ protected void randomIndexTemplate() { // otherwise, use it, it has assertions and so on that can find bugs. SuppressCodecs annotation = getClass().getAnnotation(SuppressCodecs.class); if (annotation != null && annotation.value().length == 1 && "*".equals(annotation.value()[0])) { - randomSettingsBuilder.put("index.codec", randomFrom(CODECS)); + randomSettingsBuilder.put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)); } else { randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC); } From 05b6647bf3af9556b68857ce14c131256bcd22f9 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 28 Jul 2023 14:12:22 -0400 Subject: [PATCH 182/186] [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE (#8195) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 5 +- .../opensearch/gradle/JavaPackageType.java | 15 ++ .../org/opensearch/gradle/JavaVariant.java | 197 ++++++++++++++++++ .../main/java/org/opensearch/gradle/Jdk.java | 183 +--------------- .../main/java/org/opensearch/gradle/Jre.java | 18 ++ .../opensearch/gradle/JreDownloadPlugin.java | 147 +++++++++++++ .../gradle/OpenSearchDistribution.java | 12 +- .../opensearch/gradle/VersionProperties.java | 4 + .../InternalDistributionDownloadPlugin.java | 7 +- .../gradle/test/DistroTestPlugin.java | 21 +- .../opensearch.jre-download.properties | 12 ++ .../DistributionDownloadPluginTests.java | 63 ++++-- distribution/archives/build.gradle | 55 +++-- distribution/build.gradle | 61 +++++- distribution/src/bin/opensearch-env | 9 +- .../org/opensearch/monitor/jvm/JvmInfo.java | 9 +- .../main/java/org/opensearch/node/Node.java | 2 +- settings.gradle | 1 + 19 files changed, 568 insertions(+), 254 deletions(-) create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/Jre.java create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 12dae4fca545e..2f20b3b8d7b51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) - Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) +- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0baf626142238..1dce1f8a75035 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -264,7 +264,10 @@ This repository is split into many top level directories. The most important one ### `distribution` -Builds our tar and zip archives and our rpm and deb packages. +Builds our tar and zip archives and our rpm and deb packages. There are several flavors of the distributions, with the classifier included in the name of the final deliverable (archive or package): + - default (no classifier), the distribution with bundled JDK + - `-no-jdk-` - the distribution without bundled JDK/JRE, assumes the JDK/JRE is going to be pre-installed on the target systems + - `-jre-` - the distribution bundled with JRE (smaller footprint), supported as experimental feature for some platforms ### `libs` diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java new file mode 100644 index 0000000000000..2acc335d80df0 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +public enum JavaPackageType { + NONE, + JRE, + JDK +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java new file mode 100644 index 0000000000000..5f576984627a8 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.gradle.api.Buildable; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.TaskDependency; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +abstract class JavaVariant implements Buildable, Iterable { + + private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList( + Arrays.asList("aarch64", "x64", "s390x", "ppc64le") + ); + private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); + private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( + Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") + ); + private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); + private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?"); + + private final String name; + private final Configuration configuration; + + private final Property vendor; + private final Property version; + private final Property platform; + private final Property architecture; + private String baseVersion; + private String major; + private String build; + private String hash; + + JavaVariant(String name, Configuration configuration, ObjectFactory objectFactory) { + this.name = name; + this.configuration = configuration; + this.vendor = objectFactory.property(String.class); + this.version = objectFactory.property(String.class); + this.platform = objectFactory.property(String.class); + this.architecture = objectFactory.property(String.class); + } + + public String getName() { + return name; + } + + public String getVendor() { + return vendor.get(); + } + + public void setVendor(final String vendor) { + if (ALLOWED_VENDORS.contains(vendor) == false) { + throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS); + } + this.vendor.set(vendor); + } + + public String getVersion() { + return version.get(); + } + + public void setVersion(String version) { + if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) { + throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]"); + } + parseVersion(version); + this.version.set(version); + } + + public String getPlatform() { + return platform.get(); + } + + public void setPlatform(String platform) { + if (ALLOWED_PLATFORMS.contains(platform) == false) { + throw new IllegalArgumentException( + "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS + ); + } + this.platform.set(platform); + } + + public String getArchitecture() { + return architecture.get(); + } + + public void setArchitecture(final String architecture) { + String jdkArchitecture = translateJdkArchitecture(architecture); + if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) { + throw new IllegalArgumentException( + "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES + ); + } + this.architecture.set(jdkArchitecture); + } + + public String getBaseVersion() { + return baseVersion; + } + + public String getMajor() { + return major; + } + + public String getBuild() { + return build; + } + + public String getHash() { + return hash; + } + + public String getPath() { + return configuration.getSingleFile().toString(); + } + + public String getConfigurationName() { + return configuration.getName(); + } + + @Override + public String toString() { + return getPath(); + } + + @Override + public TaskDependency getBuildDependencies() { + return configuration.getBuildDependencies(); + } + + // internal, make this jdks configuration unmodifiable + void finalizeValues() { + if (version.isPresent() == false) { + throw new IllegalArgumentException("version not specified for jdk [" + name + "]"); + } + if (platform.isPresent() == false) { + throw new IllegalArgumentException("platform not specified for jdk [" + name + "]"); + } + if (vendor.isPresent() == false) { + throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); + } + if (architecture.isPresent() == false) { + throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); + } + version.finalizeValue(); + platform.finalizeValue(); + vendor.finalizeValue(); + architecture.finalizeValue(); + } + + @Override + public Iterator iterator() { + return configuration.iterator(); + } + + private void parseVersion(String version) { + // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build] + // Note the "patch" version is not yet handled here, as it has not yet been used by java. + Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version); + if (jdkVersionMatcher.matches() == false) { + // Try again with the pre-Java9 version format + jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version); + + if (jdkVersionMatcher.matches() == false) { + throw new IllegalArgumentException("Malformed jdk version [" + version + "]"); + } + } + + baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); + major = jdkVersionMatcher.group(1); + build = jdkVersionMatcher.group(3); + hash = jdkVersionMatcher.group(5); + } + + private String translateJdkArchitecture(String architecture) { + /* + * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. + */ + return "arm64".equals(architecture) ? "aarch64" : architecture; + } + +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java index 06e857744be2d..3218abe726639 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java @@ -32,140 +32,13 @@ package org.opensearch.gradle; -import org.gradle.api.Buildable; import org.gradle.api.artifacts.Configuration; import org.gradle.api.model.ObjectFactory; -import org.gradle.api.provider.Property; -import org.gradle.api.tasks.TaskDependency; import org.gradle.internal.os.OperatingSystem; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class Jdk implements Buildable, Iterable { - - private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList( - Arrays.asList("aarch64", "x64", "s390x", "ppc64le") - ); - private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); - private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( - Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") - ); - private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); - private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?"); - - private final String name; - private final Configuration configuration; - - private final Property vendor; - private final Property version; - private final Property platform; - private final Property architecture; - private String baseVersion; - private String major; - private String build; - private String hash; - +public class Jdk extends JavaVariant { Jdk(String name, Configuration configuration, ObjectFactory objectFactory) { - this.name = name; - this.configuration = configuration; - this.vendor = objectFactory.property(String.class); - this.version = objectFactory.property(String.class); - this.platform = objectFactory.property(String.class); - this.architecture = objectFactory.property(String.class); - } - - public String getName() { - return name; - } - - public String getVendor() { - return vendor.get(); - } - - public void setVendor(final String vendor) { - if (ALLOWED_VENDORS.contains(vendor) == false) { - throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS); - } - this.vendor.set(vendor); - } - - public String getVersion() { - return version.get(); - } - - public void setVersion(String version) { - if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) { - throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]"); - } - parseVersion(version); - this.version.set(version); - } - - public String getPlatform() { - return platform.get(); - } - - public void setPlatform(String platform) { - if (ALLOWED_PLATFORMS.contains(platform) == false) { - throw new IllegalArgumentException( - "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS - ); - } - this.platform.set(platform); - } - - public String getArchitecture() { - return architecture.get(); - } - - public void setArchitecture(final String architecture) { - String jdkArchitecture = translateJdkArchitecture(architecture); - if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) { - throw new IllegalArgumentException( - "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES - ); - } - this.architecture.set(jdkArchitecture); - } - - public String getBaseVersion() { - return baseVersion; - } - - public String getMajor() { - return major; - } - - public String getBuild() { - return build; - } - - public String getHash() { - return hash; - } - - public String getPath() { - return configuration.getSingleFile().toString(); - } - - public String getConfigurationName() { - return configuration.getName(); - } - - @Override - public String toString() { - return getPath(); - } - - @Override - public TaskDependency getBuildDependencies() { - return configuration.getBuildDependencies(); + super(name, configuration, objectFactory); } public Object getBinJavaPath() { @@ -190,56 +63,4 @@ private String getHomeRoot() { boolean isOSX = "mac".equals(getPlatform()) || "darwin".equals(getPlatform()); return getPath() + (isOSX ? "/Contents/Home" : ""); } - - // internal, make this jdks configuration unmodifiable - void finalizeValues() { - if (version.isPresent() == false) { - throw new IllegalArgumentException("version not specified for jdk [" + name + "]"); - } - if (platform.isPresent() == false) { - throw new IllegalArgumentException("platform not specified for jdk [" + name + "]"); - } - if (vendor.isPresent() == false) { - throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); - } - if (architecture.isPresent() == false) { - throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); - } - version.finalizeValue(); - platform.finalizeValue(); - vendor.finalizeValue(); - architecture.finalizeValue(); - } - - @Override - public Iterator iterator() { - return configuration.iterator(); - } - - private void parseVersion(String version) { - // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build] - // Note the "patch" version is not yet handled here, as it has not yet been used by java. - Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version); - if (jdkVersionMatcher.matches() == false) { - // Try again with the pre-Java9 version format - jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version); - - if (jdkVersionMatcher.matches() == false) { - throw new IllegalArgumentException("Malformed jdk version [" + version + "]"); - } - } - - baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); - major = jdkVersionMatcher.group(1); - build = jdkVersionMatcher.group(3); - hash = jdkVersionMatcher.group(5); - } - - private String translateJdkArchitecture(String architecture) { - /* - * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. - */ - return "arm64".equals(architecture) ? "aarch64" : architecture; - } - } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jre.java b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java new file mode 100644 index 0000000000000..473bfc4860b80 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.model.ObjectFactory; + +public class Jre extends JavaVariant { + Jre(String name, Configuration configuration, ObjectFactory objectFactory) { + super(name, configuration, objectFactory); + } +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java new file mode 100644 index 0000000000000..5a00f41f07a60 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform; +import org.opensearch.gradle.transform.UnzipTransform; +import org.gradle.api.GradleException; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.dsl.RepositoryHandler; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.internal.artifacts.ArtifactAttributes; + +public class JreDownloadPlugin implements Plugin { + public static final String VENDOR_ADOPTIUM = "adoptium"; + + private static final String REPO_NAME_PREFIX = "jre_repo_"; + private static final String EXTENSION_NAME = "jres"; + public static final String JRE_TRIMMED_PREFIX = "jdk-?\\d.*-jre"; + + @Override + public void apply(Project project) { + Attribute jreAttribute = Attribute.of("jre", Boolean.class); + project.getDependencies().getAttributesSchema().attribute(jreAttribute); + project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE); + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE) + .attribute(jreAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jreAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX)); + }); + + ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); + project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName()) + .attribute(jreAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jreAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX)); + }); + + NamedDomainObjectContainer jresContainer = project.container(Jre.class, name -> { + Configuration configuration = project.getConfigurations().create("jre_" + name); + configuration.setCanBeConsumed(false); + configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + configuration.getAttributes().attribute(jreAttribute, true); + Jre jre = new Jre(name, configuration, project.getObjects()); + configuration.defaultDependencies(dependencies -> { + jre.finalizeValues(); + setupRepository(project, jre); + dependencies.add(project.getDependencies().create(dependencyNotation(jre))); + }); + return jre; + }); + project.getExtensions().add(EXTENSION_NAME, jresContainer); + } + + private void setupRepository(Project project, Jre jre) { + RepositoryHandler repositories = project.getRepositories(); + + /* + * Define the appropriate repository for the given JRE vendor and version + * + * For Oracle/OpenJDK/AdoptOpenJDK we define a repository per-version. + */ + String repoName = REPO_NAME_PREFIX + jre.getVendor() + "_" + jre.getVersion(); + String repoUrl; + String artifactPattern; + + if (jre.getVendor().equals(VENDOR_ADOPTIUM)) { + repoUrl = "https://github.com/adoptium/temurin" + jre.getMajor() + "-binaries/releases/download/"; + + if (jre.getMajor().equals("8")) { + // JDK-8 updates are always suffixed with 'U' (fe OpenJDK8U). + artifactPattern = "jdk" + + jre.getBaseVersion() + + "-" + + jre.getBuild() + + "/OpenJDK" + + jre.getMajor() + + "U" + + "-jre_[classifier]_[module]_hotspot_" + + jre.getBaseVersion() + + jre.getBuild() + + ".[ext]"; + } else { + // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17). + // To distinguish between those, the GA releases have only major version component (fe 17+32), + // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of + // version separator '.' should be enough. + artifactPattern = "jdk-" + jre.getBaseVersion() + "+" + jre.getBuild() + "/OpenJDK" + jre.getMajor() + // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release + + (jre.getBaseVersion().contains(".") || jre.getBaseVersion().matches("^2\\d+$") ? "U" : "") + + "-jre_[classifier]_[module]_hotspot_" + + jre.getBaseVersion() + + "_" + + jre.getBuild() + + ".[ext]"; + } + } else { + throw new GradleException("Unknown JDK vendor [" + jre.getVendor() + "]"); + } + + // Define the repository if we haven't already + if (repositories.findByName(repoName) == null) { + repositories.ivy(repo -> { + repo.setName(repoName); + repo.setUrl(repoUrl); + repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact); + repo.patternLayout(layout -> layout.artifact(artifactPattern)); + repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeGroup(groupName(jre))); + }); + } + } + + @SuppressWarnings("unchecked") + public static NamedDomainObjectContainer getContainer(Project project) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(EXTENSION_NAME); + } + + private static String dependencyNotation(Jre jre) { + String platformDep = jre.getPlatform().equals("darwin") || jre.getPlatform().equals("mac") ? "mac" : jre.getPlatform(); + String extension = jre.getPlatform().equals("windows") ? "zip" : "tar.gz"; + + return groupName(jre) + ":" + platformDep + ":" + jre.getBaseVersion() + ":" + jre.getArchitecture() + "@" + extension; + } + + private static String groupName(Jre jre) { + return jre.getVendor() + "_" + jre.getMajor() + "_jre"; + } + +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java index 968bd13bd4011..0575c23fee9f6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java @@ -101,7 +101,7 @@ public boolean shouldExtract() { private final Property version; private final Property type; private final Property platform; - private final Property bundledJdk; + private final Property bundledJdk; private final Property failIfUnavailable; private final Configuration extracted; @@ -120,7 +120,7 @@ public boolean shouldExtract() { this.type = objectFactory.property(Type.class); this.type.convention(Type.ARCHIVE); this.platform = objectFactory.property(Platform.class); - this.bundledJdk = objectFactory.property(Boolean.class); + this.bundledJdk = objectFactory.property(JavaPackageType.class); this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true); this.extracted = extractedConfiguration; } @@ -154,8 +154,8 @@ public void setType(Type type) { this.type.set(type); } - public boolean getBundledJdk() { - return bundledJdk.getOrElse(true); + public JavaPackageType getBundledJdk() { + return bundledJdk.getOrElse(JavaPackageType.JDK); } public boolean isDocker() { @@ -163,7 +163,7 @@ public boolean isDocker() { return type == Type.DOCKER; } - public void setBundledJdk(Boolean bundledJdk) { + public void setBundledJdk(JavaPackageType bundledJdk) { this.bundledJdk.set(bundledJdk); } @@ -266,7 +266,7 @@ void finalizeValues() { } if (bundledJdk.isPresent() == false) { - bundledJdk.set(true); + bundledJdk.set(JavaPackageType.JDK); } version.finalizeValue(); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java index 7c942358e12c3..4d8b62d95dff1 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java @@ -76,6 +76,10 @@ public static String getBundledJdk(final String platform) { return getBundledJdk(platform, null); } + public static String getBundledJre(final String platform, final String arch) { + return getBundledJdk(platform, arch); + } + public static String getBundledJdkVendor() { return bundledJdkVendor; } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java index f4368b1cecc59..6a54612320c6c 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -37,6 +37,7 @@ import org.opensearch.gradle.DistributionDependency; import org.opensearch.gradle.DistributionDownloadPlugin; import org.opensearch.gradle.DistributionResolution; +import org.opensearch.gradle.JavaPackageType; import org.opensearch.gradle.OpenSearchDistribution; import org.opensearch.gradle.Version; import org.opensearch.gradle.VersionProperties; @@ -99,7 +100,7 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer< resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> { BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion())); if (unreleasedInfo != null) { - if (!distribution.getBundledJdk()) { + if (distribution.getBundledJdk() == JavaPackageType.NONE) { throw new GradleException( "Configuring a snapshot bwc distribution ('" + distribution.getName() @@ -167,8 +168,10 @@ private static String distributionProjectName(OpenSearchDistribution distributio ? "" : "-" + architecture.toString().toLowerCase(); - if (distribution.getBundledJdk() == false) { + if (distribution.getBundledJdk() == JavaPackageType.NONE) { projectName += "no-jdk-"; + } else if (distribution.getBundledJdk() == JavaPackageType.JRE) { + projectName += "jre-"; } switch (distribution.getType()) { case ARCHIVE: diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index d32172758cfce..a311d89583acb 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -34,6 +34,7 @@ import org.opensearch.gradle.Architecture; import org.opensearch.gradle.DistributionDownloadPlugin; +import org.opensearch.gradle.JavaPackageType; import org.opensearch.gradle.OpenSearchDistribution; import org.opensearch.gradle.Jdk; import org.opensearch.gradle.JdkDownloadPlugin; @@ -71,6 +72,7 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; @@ -137,7 +139,7 @@ public void apply(Project project) { } if ((distribution.getType() == OpenSearchDistribution.Type.DEB || distribution.getType() == OpenSearchDistribution.Type.RPM) - && distribution.getBundledJdk()) { + && distribution.getBundledJdk() != JavaPackageType.NONE) { for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { if (version.before("6.3.0")) { continue; // before opening xpack @@ -379,8 +381,8 @@ private List configureDistributions(Project project) { OpenSearchDistribution.Type.RPM, OpenSearchDistribution.Type.DOCKER )) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false) { + for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) { + if (bundledJdk == JavaPackageType.NONE) { // We'll never publish an ARM (arm64) build without a bundled JDK. if (architecture == Architecture.ARM64) { continue; @@ -403,8 +405,8 @@ private List configureDistributions(Project project) { OpenSearchDistribution.Platform.LINUX, OpenSearchDistribution.Platform.WINDOWS )) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false && architecture != Architecture.X64) { + for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) { + if (bundledJdk == JavaPackageType.NONE && architecture != Architecture.X64) { // We will never publish distributions for non-x86 (amd64) platforms // without a bundled JDK continue; @@ -432,7 +434,7 @@ private static OpenSearchDistribution createDistro( Architecture architecture, OpenSearchDistribution.Type type, OpenSearchDistribution.Platform platform, - boolean bundledJdk, + JavaPackageType bundledJdk, String version ) { String name = distroId(type, platform, bundledJdk, architecture) + "-" + version; @@ -466,11 +468,12 @@ private static boolean isWindows(Project project) { private static String distroId( OpenSearchDistribution.Type type, OpenSearchDistribution.Platform platform, - boolean bundledJdk, + JavaPackageType bundledJdk, Architecture architecture ) { - return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk") - + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); + return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk != JavaPackageType.NONE + ? (bundledJdk == JavaPackageType.JDK ? "" : "-jre") + : "-no-jdk") + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); } private static String destructiveDistroTestTaskName(OpenSearchDistribution distro) { diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties new file mode 100644 index 0000000000000..e9253488ffbeb --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties @@ -0,0 +1,12 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +implementation-class=org.opensearch.gradle.JreDownloadPlugin diff --git a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java index 1a9647573f948..e439e8a047f34 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java @@ -45,6 +45,7 @@ import java.util.Arrays; import java.util.TreeSet; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.StringContains.containsString; public class DistributionDownloadPluginTests extends GradleUnitTestCase { @@ -76,7 +77,14 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase { ); public void testVersionDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", null, Type.ARCHIVE, Platform.LINUX, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + null, + Type.ARCHIVE, + Platform.LINUX, + JavaPackageType.JDK + ); assertEquals(distro.getVersion(), VersionProperties.getOpenSearch()); } @@ -123,18 +131,32 @@ public void testBadVersionFormat() { "badversion", Type.ARCHIVE, Platform.LINUX, - true, + JavaPackageType.JDK, "Invalid version format: 'badversion'" ); } public void testTypeDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", null, Platform.LINUX, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + null, + Platform.LINUX, + JavaPackageType.JDK + ); assertEquals(distro.getType(), Type.ARCHIVE); } public void testPlatformDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, null, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + Type.ARCHIVE, + null, + JavaPackageType.JDK + ); assertEquals(distro.getPlatform(), OpenSearchDistribution.CURRENT_PLATFORM); } @@ -151,8 +173,15 @@ public void testPlatformForIntegTest() { } public void testBundledJdkDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, true); - assertTrue(distro.getBundledJdk()); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + Type.ARCHIVE, + Platform.LINUX, + JavaPackageType.JDK + ); + assertThat(distro.getBundledJdk(), equalTo(JavaPackageType.JDK)); } public void testBundledJdkForIntegTest() { @@ -162,7 +191,7 @@ public void testBundledJdkForIntegTest() { "5.0.0", Type.INTEG_TEST_ZIP, null, - true, + JavaPackageType.JDK, "bundledJdk cannot be set on opensearch distribution [testdistro]" ); } @@ -178,7 +207,7 @@ public void testLocalCurrentVersionIntegTestZip() { public void testLocalCurrentVersionArchives() { for (Platform platform : Platform.values()) { - for (boolean bundledJdk : new boolean[] { true, false }) { + for (JavaPackageType bundledJdk : JavaPackageType.values()) { for (Architecture architecture : Architecture.values()) { // create a new project in each iteration, so that we know we are resolving the only additional project being created Project project = createProject(BWC_MINOR, true); @@ -204,7 +233,7 @@ public void testLocalCurrentVersionArchives() { public void testLocalCurrentVersionPackages() { for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { - for (boolean bundledJdk : new boolean[] { true, false }) { + for (JavaPackageType bundledJdk : JavaPackageType.values()) { Project project = createProject(BWC_MINOR, true); String projectName = projectName(packageType.toString(), bundledJdk); Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build(); @@ -219,7 +248,7 @@ public void testLocalCurrentVersionPackages() { public void testLocalBwcArchives() { for (Platform platform : Platform.values()) { // note: no non bundled jdk for bwc - String configName = projectName(platform.toString(), true); + String configName = projectName(platform.toString(), JavaPackageType.JDK); configName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); checkBwc("minor", configName, BWC_MINOR_VERSION, Type.ARCHIVE, platform, BWC_MINOR, true); @@ -232,7 +261,7 @@ public void testLocalBwcArchives() { public void testLocalBwcPackages() { for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { // note: no non bundled jdk for bwc - String configName = projectName(packageType.toString(), true); + String configName = projectName(packageType.toString(), JavaPackageType.JDK); checkBwc("minor", configName, BWC_MINOR_VERSION, packageType, null, BWC_MINOR, true); checkBwc("staged", configName, BWC_STAGED_VERSION, packageType, null, BWC_STAGED, true); @@ -247,7 +276,7 @@ private void assertDistroError( String version, Type type, Platform platform, - Boolean bundledJdk, + JavaPackageType bundledJdk, String message ) { IllegalArgumentException e = expectThrows( @@ -263,7 +292,7 @@ private OpenSearchDistribution createDistro( String version, Type type, Platform platform, - Boolean bundledJdk + JavaPackageType bundledJdk ) { NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); return distros.create(name, distro -> { @@ -289,7 +318,7 @@ private OpenSearchDistribution checkDistro( String version, Type type, Platform platform, - Boolean bundledJdk + JavaPackageType bundledJdk ) { OpenSearchDistribution distribution = createDistro(project, name, version, type, platform, bundledJdk); distribution.finalizeValues(); @@ -315,7 +344,7 @@ private void checkBwc( Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build(); archiveProject.getConfigurations().create(config); archiveProject.getArtifacts().add(config, new File("doesnotmatter")); - final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, true); + final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, JavaPackageType.JDK); distro.setArchitecture(Architecture.current()); checkPlugin(project); } @@ -335,7 +364,7 @@ private Project createProject(BwcVersions bwcVersions, boolean isInternal) { return project; } - private static String projectName(String base, boolean bundledJdk) { - return bundledJdk ? base : ("no-jdk-" + base); + private static String projectName(String base, JavaPackageType bundledJdk) { + return (bundledJdk == JavaPackageType.JDK) ? base : ((bundledJdk == JavaPackageType.NONE) ? ("no-jdk-" + base) : "jre-" + base); } } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index e7137127053e2..161b8008525b4 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -28,9 +28,11 @@ * under the License. */ +import org.opensearch.gradle.JavaPackageType + apply plugin: 'opensearch.internal-distribution-archive-setup' -CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean jdk) { +CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, JavaPackageType java) { return copySpec { into("opensearch-${version}") { into('lib') { @@ -39,19 +41,23 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla into('config') { dirMode 0750 fileMode 0660 - with configFiles(distributionType, jdk) + with configFiles(distributionType, java) from { dirMode 0750 jvmOptionsDir.getParent() } } into('bin') { - with binFiles(distributionType, jdk) + with binFiles(distributionType, java) } - if (jdk) { + if (java == JavaPackageType.JDK) { into("darwin".equals(platform) ? 'jdk.app' : 'jdk') { with jdkFiles(project, platform, architecture) } + } else if (java == JavaPackageType.JRE) { + into("darwin".equals(platform) ? 'jre.app' : 'jre') { + with jreFiles(project, platform, architecture) + } } into('') { from { @@ -73,7 +79,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla rename { 'LICENSE.txt' } } - with noticeFile(jdk) + with noticeFile(java) into('modules') { with modulesFiles } @@ -84,77 +90,84 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla distribution_archives { integTestZip { content { - archiveFiles(transportModulesFiles, 'zip', null, 'x64', false) + archiveFiles(transportModulesFiles, 'zip', null, 'x64', JavaPackageType.NONE) } } darwinTar { archiveClassifier = 'darwin-x64' content { - archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', true) + archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.JDK) } } darwinArm64Tar { archiveClassifier = 'darwin-arm64' content { - archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', true) + archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.JDK) } } noJdkDarwinTar { archiveClassifier = 'no-jdk-darwin-x64' content { - archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', false) + archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.NONE) } } noJdkDarwinArm64Tar { archiveClassifier = 'no-jdk-darwin-arm64' content { - archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', false) + archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.NONE) } } freebsdTar { archiveClassifier = 'freebsd-x64' content { - archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false) + archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE) } } noJdkFreebsdTar { archiveClassifier = 'no-jdk-freebsd-x64' content { - archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false) + archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE) } } linuxArm64Tar { archiveClassifier = 'linux-arm64' content { - archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', true) + archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.JDK) } } noJdkLinuxArm64Tar { archiveClassifier = 'no-jdk-linux-arm64' content { - archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', false) + archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.NONE) } } linuxTar { archiveClassifier = 'linux-x64' content { - archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', true) + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JDK) } } noJdkLinuxTar { archiveClassifier = 'no-jdk-linux-x64' content { - archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', false) + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.NONE) + } + } + + jreLinuxTar { + archiveClassifier = 'jre-linux-x64' + content { + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JRE) } } @@ -163,7 +176,7 @@ distribution_archives { linuxS390xTar { archiveClassifier = 'linux-s390x' content { - archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', false) + archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', JavaPackageType.NONE) } } @@ -171,28 +184,28 @@ distribution_archives { linuxPpc64leTar { archiveClassifier = 'linux-ppc64le' content { - archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', true) + archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.JDK) } } noJdkLinuxPpc64leTar { archiveClassifier = 'no-jdk-linux-ppc64le' content { - archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', false) + archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.NONE) } } windowsZip { archiveClassifier = 'windows-x64' content { - archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', true) + archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.JDK) } } noJdkWindowsZip { archiveClassifier = 'no-jdk-windows-x64' content { - archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', false) + archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.NONE) } } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 2378ed833ead7..35ca84ca66dba 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -36,6 +36,7 @@ import org.opensearch.gradle.MavenFilteringHack import org.opensearch.gradle.NoticeTask import org.opensearch.gradle.VersionProperties import org.opensearch.gradle.info.BuildParams +import org.opensearch.gradle.JavaPackageType import java.nio.file.Files import java.nio.file.Path @@ -275,13 +276,14 @@ project(':test:external-modules').subprojects.each { Project testModule -> configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { apply plugin: 'opensearch.jdk-download' + apply plugin: 'opensearch.jre-download' apply plugin: 'opensearch.repositories' // Setup all required JDKs project.jdks { ['darwin', 'linux', 'windows'].each { platform -> (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture -> - "bundled_${platform}_${architecture}" { + "bundled_jdk_${platform}_${architecture}" { it.platform = platform it.version = VersionProperties.getBundledJdk(platform, architecture) it.vendor = VersionProperties.bundledJdkVendor @@ -291,6 +293,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } + // Setup all required JREs + project.jres { + ['darwin', 'linux', 'windows'].each { platform -> + (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture -> + "bundled_jre_${platform}_${architecture}" { + it.platform = platform + it.version = VersionProperties.getBundledJre(platform, architecture) + it.vendor = VersionProperties.bundledJdkVendor + it.architecture = architecture + } + } + } + } + // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... /***************************************************************************** * Properties to expand when copying packaging files * @@ -377,20 +393,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from buildTransportModulesTaskProvider } - configFiles = { distributionType, jdk -> + configFiles = { distributionType, java -> copySpec { with copySpec { // main config files, processed with distribution specific substitutions from '../src/config' exclude 'log4j2.properties' // this is handled separately below - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } from project(':distribution').buildLog4jConfig from project(':distribution').buildConfig } } - binFiles = { distributionType, jdk -> + binFiles = { distributionType, java -> copySpec { // non-windows files, for all distributions with copySpec { @@ -398,7 +414,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude '*.exe' exclude '*.bat' eachFile { it.setMode(0755) } - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } // windows files, only for zip if (distributionType == 'zip') { @@ -406,7 +422,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from '../src/bin' include '*.bat' filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } with copySpec { from '../src/bin' @@ -424,12 +440,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } - noticeFile = { jdk -> + noticeFile = { java -> copySpec { if (project.name == 'integ-test-zip') { from buildServerNoticeTaskProvider } else { - if (jdk) { + if (java != JavaPackageType.NONE) { from buildNoticeTaskProvider } else { from buildNoJdkNoticeTaskProvider @@ -446,7 +462,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { if ("arm64".equals(architecture)) { architecture = "aarch64" } - from project.jdks."bundled_${platform}_${architecture}" + from project.jdks."bundled_jdk_${platform}_${architecture}" exclude "demo/**" /* * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude @@ -465,6 +481,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } } + + jreFiles = { Project project, String platform, String architecture -> + return copySpec { + /* + * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. + */ + if ("arm64".equals(architecture)) { + architecture = "aarch64" + } + from project.jres."bundled_jre_${platform}_${architecture}" + exclude "demo/**" + /* + * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude + * it from the build. + */ + if ("darwin".equals(platform)) { + exclude "Contents/MacOS" + } + eachFile { FileCopyDetails details -> + if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') { + details.mode = 0755 + } + } + } + } } } @@ -583,7 +624,7 @@ subprojects { ], 'opensearch.bundled_jdk': [ - 'def': jdk ? 'true' : 'false' + 'def': jdk != JavaPackageType.NONE ? true : false ], 'license.name': [ diff --git a/distribution/src/bin/opensearch-env b/distribution/src/bin/opensearch-env index f94824576b122..cd7a0b2b4520b 100644 --- a/distribution/src/bin/opensearch-env +++ b/distribution/src/bin/opensearch-env @@ -46,7 +46,7 @@ fi # now set the classpath OPENSEARCH_CLASSPATH="$OPENSEARCH_HOME/lib/*" -# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JDK +# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JRE -> bundled JDK if [ ! -z "$OPENSEARCH_JAVA_HOME" ]; then JAVA="$OPENSEARCH_JAVA_HOME/bin/java" JAVA_TYPE="OPENSEARCH_JAVA_HOME" @@ -57,13 +57,18 @@ else if [ $OS = "darwin" ]; then # macOS bundled Java JAVA="$OPENSEARCH_HOME/jdk.app/Contents/Home/bin/java" + JAVA_TYPE="bundled jdk" elif [ $OS = "freebsd" ]; then # using FreeBSD default java from ports if JAVA_HOME is not set JAVA="/usr/local/bin/java" + JAVA_TYPE="bundled jdk" + elif [ -d "$OPENSEARCH_HOME/jre" ]; then + JAVA="$OPENSEARCH_HOME/jre/bin/java" + JAVA_TYPE="bundled jre" else JAVA="$OPENSEARCH_HOME/jdk/bin/java" + JAVA_TYPE="bundled jdk" fi - JAVA_TYPE="bundled jdk" fi if [ ! -x "$JAVA" ]; then diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java index 1acdf00ca6af0..4ce7013e1052f 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java @@ -171,7 +171,7 @@ public class JvmInfo implements ReportingService.Info { } final boolean bundledJdk = Booleans.parseBoolean(System.getProperty("opensearch.bundled_jdk", Boolean.FALSE.toString())); - final Boolean usingBundledJdk = bundledJdk ? usingBundledJdk() : null; + final Boolean usingBundledJdkOrJre = bundledJdk ? usingBundledJdkOrJre() : null; INSTANCE = new JvmInfo( JvmPid.getPid(), @@ -180,7 +180,7 @@ public class JvmInfo implements ReportingService.Info { runtimeMXBean.getVmVersion(), runtimeMXBean.getVmVendor(), bundledJdk, - usingBundledJdk, + usingBundledJdkOrJre, runtimeMXBean.getStartTime(), configuredInitialHeapSize, configuredMaxHeapSize, @@ -201,7 +201,7 @@ public class JvmInfo implements ReportingService.Info { } @SuppressForbidden(reason = "PathUtils#get") - private static boolean usingBundledJdk() { + private static boolean usingBundledJdkOrJre() { /* * We are using the bundled JDK if java.home is the jdk sub-directory of our working directory. This is because we always set * the working directory of Elasticsearch to home, and the bundled JDK is in the jdk sub-directory there. @@ -211,7 +211,8 @@ private static boolean usingBundledJdk() { if (Constants.MAC_OS_X) { return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk.app/Contents/Home").toAbsolutePath()); } else { - return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath()); + return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jre").toAbsolutePath()) + || PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath()); } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index d768165451a5a..fe630dfe27e3a 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -428,7 +428,7 @@ protected Node( Constants.JVM_VERSION ); if (jvmInfo.getBundledJdk()) { - logger.info("JVM home [{}], using bundled JDK [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk()); + logger.info("JVM home [{}], using bundled JDK/JRE [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk()); } else { logger.info("JVM home [{}]", System.getProperty("java.home")); deprecationLogger.deprecate( diff --git a/settings.gradle b/settings.gradle index b7d47cd9b745e..94f5de26f4090 100644 --- a/settings.gradle +++ b/settings.gradle @@ -53,6 +53,7 @@ List projects = [ 'distribution:archives:no-jdk-linux-ppc64le-tar', 'distribution:archives:linux-tar', 'distribution:archives:no-jdk-linux-tar', + 'distribution:archives:jre-linux-tar', 'distribution:docker', 'distribution:docker:docker-arm64-build-context', 'distribution:docker:docker-arm64-export', From b282005191bca3e3dd5b5ad5eb347dcaf296e4e5 Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Fri, 28 Jul 2023 13:48:19 -0500 Subject: [PATCH 183/186] Replace deprecated Query#rewrite(IndexReader with rewrite(IndexSearcher (#8967) * Replace deprecated Query#rewrite(IndexReader with rewrite(IndexSearcher Query#rewrite(IndexReader reader) is deprecated in Lucene 9.7 and removed in Lucene 10 in favor of Query#rewrite(IndexSearcher searcher). The latter provides LeafCollector hooks to optimize for concurrent queries. This commit cuts over usage of rewrite(IndexReader) to rewrite(IndexSearcher) for Lucene 10 compatibility and upstream query optimizations. Signed-off-by: Nicholas Walter Knize * remove stray collector.finish Signed-off-by: Nicholas Walter Knize --------- Signed-off-by: Nicholas Walter Knize --- .../join/query/HasChildQueryBuilder.java | 5 +++-- .../org/opensearch/percolator/PercolateQuery.java | 5 ++--- .../opensearch/percolator/CandidateQueryTests.java | 2 +- .../apache/lucene/queries/BlendedTermQuery.java | 6 ++++-- .../common/lucene/search/MoreLikeThisQuery.java | 8 ++++---- .../lucene/search/MultiPhrasePrefixQuery.java | 7 ++++--- .../lucene/search/function/FunctionScoreQuery.java | 10 +++++----- .../lucene/search/function/ScriptScoreQuery.java | 7 +++---- .../SortedUnsignedLongDocValuesRangeQuery.java | 5 ++--- .../index/query/DateRangeIncludingNowQuery.java | 4 ++-- .../search/OpenSearchToParentBlockJoinQuery.java | 7 +++---- .../org/opensearch/lucene/queries/MinDocQuery.java | 3 ++- .../suggest/completion/CompletionSuggester.java | 2 +- .../deps/lucene/VectorHighlighterTests.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 6 +++++- .../query/SpanMultiTermQueryBuilderTests.java | 5 +++-- .../OpenSearchToParentBlockJoinQueryTests.java | 3 ++- .../lucene/queries/MinDocQueryTests.java | 5 +++-- .../lucene/queries/SpanMatchNoDocsQueryTests.java | 14 +++----------- .../search/internal/ContextIndexSearcherTests.java | 7 +++---- 20 files changed, 56 insertions(+), 57 deletions(-) diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java index 4f26a15031dd4..b38bc5d7b45fc 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java @@ -416,11 +416,12 @@ public void visit(QueryVisitor visitor) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } + IndexReader reader = searcher.getIndexReader(); if (reader instanceof DirectoryReader) { IndexSearcher indexSearcher = new IndexSearcher(reader); indexSearcher.setQueryCache(null); diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java index 86d30f009e709..2702a02a7a951 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java @@ -32,7 +32,6 @@ package org.opensearch.percolator; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -89,8 +88,8 @@ final class PercolateQuery extends Query implements Accountable { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = candidateMatchesQuery.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = candidateMatchesQuery.rewrite(searcher); if (rewritten != candidateMatchesQuery) { return new PercolateQuery( name, diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java index 3b953fcfe65e1..1040d014483e1 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java @@ -1275,7 +1275,7 @@ private CustomQuery(Term term) { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { return new TermQuery(term); } diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 3a36a6ff103e0..9d276cfe283eb 100644 --- a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -42,6 +42,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.TermQuery; @@ -93,11 +94,12 @@ public BlendedTermQuery(Term[] terms, float[] boosts) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } + IndexReader reader = searcher.getIndexReader(); IndexReaderContext context = reader.getContext(); TermStates[] ctx = new TermStates[terms.length]; int[] docFreqs = new int[ctx.length]; diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java index b4440b85e0037..ef07f6ea8052c 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java @@ -36,12 +36,12 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.similarities.ClassicSimilarity; @@ -144,12 +144,12 @@ public boolean equals(Object obj) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } - XMoreLikeThis mlt = new XMoreLikeThis(reader, similarity == null ? new ClassicSimilarity() : similarity); + XMoreLikeThis mlt = new XMoreLikeThis(searcher.getIndexReader(), similarity == null ? new ClassicSimilarity() : similarity); mlt.setFieldNames(moreLikeFields); mlt.setAnalyzer(analyzer); diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java index 98ebb34fc040e..cc0468efb243e 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.Query; @@ -159,8 +160,8 @@ public int[] getPositions() { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } @@ -177,7 +178,7 @@ public Query rewrite(IndexReader reader) throws IOException { int position = positions.get(sizeMinus1); Set terms = new HashSet<>(); for (Term term : suffixTerms) { - getPrefixTerms(terms, term, reader); + getPrefixTerms(terms, term, searcher.getIndexReader()); if (terms.size() > maxExpansions) { break; } diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java index ebd155b8a5679..db92d4f1cba96 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java @@ -128,7 +128,7 @@ protected int doHashCode() { @Override protected ScoreFunction rewrite(IndexReader reader) throws IOException { - Query newFilter = filter.rewrite(reader); + Query newFilter = filter.rewrite(new IndexSearcher(reader)); if (newFilter == filter) { return this; } @@ -322,16 +322,16 @@ public void visit(QueryVisitor visitor) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } - Query newQ = subQuery.rewrite(reader); + Query newQ = subQuery.rewrite(searcher); ScoreFunction[] newFunctions = new ScoreFunction[functions.length]; boolean needsRewrite = (newQ != subQuery); for (int i = 0; i < functions.length; i++) { - newFunctions[i] = functions[i].rewrite(reader); + newFunctions[i] = functions[i].rewrite(searcher.getIndexReader()); needsRewrite |= (newFunctions[i] != functions[i]); } if (needsRewrite) { diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java index 8bf5fc0f89d31..07084a6f690e4 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java @@ -32,7 +32,6 @@ package org.opensearch.common.lucene.search.function; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.DocIdSetIterator; @@ -105,12 +104,12 @@ public ScriptScoreQuery( } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query newQ = subQuery.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query newQ = subQuery.rewrite(searcher); if (newQ != subQuery) { return new ScriptScoreQuery(newQ, queryName, script, scriptBuilder, minScore, indexName, shardId, indexVersion); } - return super.rewrite(reader); + return super.rewrite(searcher); } @Override diff --git a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java index 04dd669d60883..234c67cc637f1 100644 --- a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java +++ b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -87,12 +86,12 @@ public String toString(String field) { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { if (Long.compareUnsigned(lowerValue, Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG) == 0 && Long.compareUnsigned(upperValue, Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG) == 0) { return new FieldExistsQuery(field); } - return super.rewrite(reader); + return super.rewrite(searcher); } abstract SortedNumericDocValues getValues(LeafReader reader, String field) throws IOException; diff --git a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java index 2a7dbf314103d..001d77facb829 100644 --- a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java +++ b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java @@ -32,8 +32,8 @@ package org.opensearch.index.query; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; @@ -60,7 +60,7 @@ public Query getQuery() { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { return in; } diff --git a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java index 770301629d964..cd28d8b6ef04e 100644 --- a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java +++ b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java @@ -32,7 +32,6 @@ package org.opensearch.index.search; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -82,8 +81,8 @@ public ScoreMode getScoreMode() { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query innerRewrite = query.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query innerRewrite = query.rewrite(searcher); if (innerRewrite != query) { // Right now ToParentBlockJoinQuery always rewrites to a ToParentBlockJoinQuery // so the else block will never be used. It is useful in the case that @@ -97,7 +96,7 @@ public Query rewrite(IndexReader reader) throws IOException { return innerRewrite; } } - return super.rewrite(reader); + return super.rewrite(searcher); } @Override diff --git a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java index 2af0237e800c0..5c904d8a7770d 100644 --- a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java @@ -86,7 +86,8 @@ public boolean equals(Object obj) { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { + final IndexReader reader = searcher.getIndexReader(); if (Objects.equals(reader.getContext().id(), readerId) == false) { return new MinDocQuery(minDoc, reader.getContext().id()); } diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java index df18e874be9c9..fabb67d7fb841 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java @@ -104,7 +104,7 @@ protected Suggest.Suggestion query.rewrite(reader)); + IndexSearcher searcher = new IndexSearcher(reader); + Query query = queryBuilder.toQuery(createShardContext(searcher)); + RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(searcher)); assertThat(exc.getMessage(), containsString("maxClauseCount")); } finally { BooleanQuery.setMaxClauseCount(origBoolMaxClauseCount); diff --git a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java index 39160b1ac49bc..45172c946ab36 100644 --- a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java +++ b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -112,7 +113,7 @@ public void testRewrite() throws IOException { ScoreMode.Avg, "nested" ); - Query rewritten = q.rewrite(new MultiReader()); + Query rewritten = q.rewrite(new IndexSearcher(new MultiReader())); assertEquals(expected, rewritten); } } diff --git a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java index a75a1d5b2fdb7..701ba5a756b40 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java @@ -61,10 +61,11 @@ public void testBasics() { public void testRewrite() throws Exception { IndexReader reader = new MultiReader(); + IndexSearcher searcher = new IndexSearcher(reader); MinDocQuery query = new MinDocQuery(42); - Query rewritten = query.rewrite(reader); + Query rewritten = query.rewrite(searcher); QueryUtils.checkUnequal(query, rewritten); - Query rewritten2 = rewritten.rewrite(reader); + Query rewritten2 = rewritten.rewrite(searcher); assertSame(rewritten, rewritten2); } diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java index 727377c9a5d53..81f4290b6588e 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java @@ -56,19 +56,11 @@ public class SpanMatchNoDocsQueryTests extends OpenSearchTestCase { public void testSimple() throws Exception { - Directory dir = newDirectory(); - IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig()); - IndexReader ir = DirectoryReader.open(iw); - SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason"); assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")"); - Query rewrite = query.rewrite(ir); + Query rewrite = query.rewrite((IndexSearcher) null); assertTrue(rewrite instanceof SpanMatchNoDocsQuery); assertEquals(rewrite.toString(), "SpanMatchNoDocsQuery(\"a good reason\")"); - - iw.close(); - ir.close(); - dir.close(); } public void testQuery() throws Exception { @@ -101,7 +93,7 @@ public void testQuery() throws Exception { assertEquals(searcher.count(orQuery), 1); hits = searcher.search(orQuery, 1000).scoreDocs; assertEquals(1, hits.length); - Query rewrite = orQuery.rewrite(ir); + Query rewrite = orQuery.rewrite(searcher); assertEquals(rewrite, orQuery); SpanNearQuery nearQuery = new SpanNearQuery( @@ -112,7 +104,7 @@ public void testQuery() throws Exception { assertEquals(searcher.count(nearQuery), 0); hits = searcher.search(nearQuery, 1000).scoreDocs; assertEquals(0, hits.length); - rewrite = nearQuery.rewrite(ir); + rewrite = nearQuery.rewrite(searcher); assertEquals(rewrite, nearQuery); iw.close(); diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index f3907355ac6ec..823fc6b463906 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -40,7 +40,6 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; @@ -466,12 +465,12 @@ public String toString(String field) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query queryRewritten = query.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query queryRewritten = query.rewrite(searcher); if (query != queryRewritten) { return new CreateScorerOnceQuery(queryRewritten); } - return super.rewrite(reader); + return super.rewrite(searcher); } @Override From 36c37ace30cdf9d2f92176a3aa38b60865c33056 Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Fri, 28 Jul 2023 14:13:02 -0500 Subject: [PATCH 184/186] [Refactor] XContentFactory contentType introspection to MediaTypeParserRegistry (#8826) XContentFactory is tightly coupled to concrete XContentType. This commit builds on the MediaType abstractions, specifically MediaTypeParserRegistry, to decouple contentType introspection (e.g., determining type from byte streams) from concrete XContentTypes. This enables downstream extensions (e.g., serverless or cloud native implementations) to register their own custom XContentType and define the serialization format for proper content introspection. This also removes the tight coupling of :libs:opensearch-x-content with abstract interface contracts further enabling modularity. Signed-off-by: Nicholas Walter Knize --- .../client/indices/CreateIndexRequest.java | 2 +- .../indices/PutIndexTemplateRequest.java | 3 +- .../client/indices/PutMappingRequest.java | 3 +- .../client/AbstractRequestTestCase.java | 3 +- .../client/AbstractResponseTestCase.java | 3 +- .../client/core/GetSourceResponseTests.java | 4 +- .../indices/CloseIndexResponseTests.java | 5 +- .../GetIndexTemplatesResponseTests.java | 4 +- .../opensearch/core/xcontent/MediaType.java | 10 + .../core/xcontent/MediaTypeRegistry.java | 215 +++++++++++++ .../common/xcontent/XContentFactory.java | 282 +----------------- .../common/xcontent/XContentType.java | 105 +++++++ .../xcontent/json/JsonXContentGenerator.java | 6 +- .../ingest/useragent/UserAgentParser.java | 3 +- .../mustache/SearchTemplateResponse.java | 6 +- .../TransportSearchTemplateAction.java | 4 +- .../SearchTemplateRequestXContentTests.java | 10 +- .../mustache/SearchTemplateResponseTests.java | 9 +- .../percolator/PercolateQueryBuilder.java | 22 +- .../DiscountedCumulativeGainTests.java | 4 +- .../rankeval/ExpectedReciprocalRankTests.java | 4 +- .../rankeval/MeanReciprocalRankTests.java | 4 +- .../index/rankeval/PrecisionAtKTests.java | 4 +- .../index/rankeval/RankEvalResponseTests.java | 4 +- .../index/rankeval/RatedDocumentTests.java | 4 +- .../index/rankeval/RatedRequestsTests.java | 4 +- .../index/rankeval/RecallAtKTests.java | 4 +- .../AbstractBulkByQueryRestHandler.java | 4 +- .../aggregations/FiltersAggsRewriteIT.java | 4 +- .../create/CreateSnapshotRequest.java | 3 +- .../action/admin/indices/alias/Alias.java | 3 +- .../indices/alias/IndicesAliasesRequest.java | 3 +- .../mapping/put/PutMappingRequest.java | 3 +- .../opensearch/action/index/IndexRequest.java | 6 +- .../termvectors/TermVectorsRequest.java | 4 +- .../action/update/UpdateRequest.java | 6 +- .../cluster/metadata/AliasMetadata.java | 3 +- .../cluster/metadata/AliasValidator.java | 11 +- .../cluster/metadata/IndexMetadata.java | 4 +- .../common/compress/CompressorFactory.java | 8 +- .../opensearch/common/settings/Settings.java | 8 +- .../common/xcontent/XContentHelper.java | 53 +--- .../gateway/MetadataStateFormat.java | 10 +- .../gateway/PersistedClusterStateService.java | 12 +- .../opensearch/index/get/ShardGetService.java | 8 +- .../index/mapper/SourceFieldMapper.java | 6 +- .../index/query/WrapperQueryBuilder.java | 7 +- .../functionscore/DecayFunctionBuilder.java | 5 +- .../reindex/ClientScrollableHitSource.java | 4 +- .../index/reindex/ReindexRequest.java | 4 +- .../opensearch/index/shard/IndexShard.java | 4 +- .../opensearch/indices/IndicesService.java | 4 +- .../blobstore/ChecksumBlobStoreFormat.java | 10 +- .../opensearch/rest/AbstractRestChannel.java | 8 +- .../action/document/RestGetSourceAction.java | 4 +- .../CompletionSuggestionBuilder.java | 6 +- .../suggest/phrase/PhraseSuggester.java | 5 +- .../org/opensearch/tasks/RawTaskStatus.java | 4 +- .../java/org/opensearch/tasks/TaskResult.java | 4 +- .../opensearch/tasks/TaskResultsService.java | 4 +- .../node/tasks/TransportTasksActionTests.java | 4 +- .../reroute/ClusterRerouteRequestTests.java | 4 +- .../indices/alias/AliasActionsTests.java | 4 +- .../create/CreateIndexRequestTests.java | 8 +- .../rollover/RolloverRequestTests.java | 3 +- .../put/PutIndexTemplateRequestTests.java | 5 +- .../action/bulk/BulkRequestTests.java | 16 +- .../action/explain/ExplainResponseTests.java | 4 +- .../MergedFieldCapabilitiesResponseTests.java | 4 +- .../action/get/MultiGetRequestTests.java | 2 +- .../action/get/MultiGetResponseTests.java | 3 +- .../action/support/IndicesOptionsTests.java | 8 +- .../AbstractShapeBuilderTestCase.java | 4 +- .../common/xcontent/BaseXContentTestCase.java | 3 +- .../common/xcontent/XContentFactoryTests.java | 33 +- .../builder/XContentBuilderTests.java | 72 ++--- .../common/xcontent/cbor/JsonVsCborTests.java | 5 +- .../xcontent/smile/JsonVsSmileTests.java | 5 +- .../AbstractXContentFilteringTestCase.java | 3 +- .../index/mapper/SourceFieldMapperTests.java | 6 +- .../index/query/BoolQueryBuilderTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 4 +- .../snapshots/blobstore/FileInfoTests.java | 6 +- ...oteStoreShardShallowCopySnapshotTests.java | 4 +- .../ingest/IngestMetadataTests.java | 4 +- .../PersistentTasksCustomMetadataTests.java | 3 +- .../indices/RestGetAliasesActionTests.java | 16 +- .../org/opensearch/script/ScriptTests.java | 5 +- .../opensearch/script/StoredScriptTests.java | 30 +- .../search/NestedIdentityTests.java | 4 +- .../AggregatorFactoriesTests.java | 4 +- .../aggregations/bucket/FiltersTests.java | 12 +- .../support/IncludeExcludeTests.java | 6 +- .../builder/SearchSourceBuilderTests.java | 10 +- .../highlight/HighlightBuilderTests.java | 6 +- .../highlight/HighlightFieldTests.java | 4 +- .../internal/ShardSearchRequestTests.java | 3 +- .../rescore/QueryRescorerBuilderTests.java | 4 +- .../searchafter/SearchAfterBuilderTests.java | 5 +- .../search/slice/SliceBuilderTests.java | 4 +- .../search/sort/AbstractSortTestCase.java | 4 +- .../search/sort/NestedSortBuilderTests.java | 4 +- .../AbstractSuggestionBuilderTestCase.java | 4 +- .../search/suggest/SuggestBuilderTests.java | 4 +- .../phrase/DirectCandidateGeneratorTests.java | 4 +- .../phrase/SmoothingModelTestCase.java | 4 +- .../org/opensearch/tasks/TaskResultTests.java | 4 +- .../index/RandomCreateIndexGenerator.java | 4 +- .../index/engine/TranslogHandler.java | 10 +- .../search/RandomSearchRequestGenerator.java | 2 +- .../aggregations/BaseAggregationTestCase.java | 6 +- .../BasePipelineAggregationTestCase.java | 4 +- .../test/AbstractXContentTestCase.java | 5 +- .../opensearch/test/OpenSearchTestCase.java | 4 +- .../org/opensearch/test/RandomObjects.java | 4 +- .../opensearch/test/XContentTestUtils.java | 3 +- .../yaml/ClientYamlTestExecutionContext.java | 4 +- .../test/rest/yaml/ObjectPathTests.java | 7 +- 118 files changed, 711 insertions(+), 667 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java index 16915b32c16fe..cffed98fce3aa 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java @@ -187,7 +187,7 @@ public CreateIndexRequest mapping(XContentBuilder source) { */ public CreateIndexRequest mapping(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); builder.map(source); return mapping(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java index fed958955af99..09dbbd63b9479 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java @@ -48,6 +48,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -267,7 +268,7 @@ public PutIndexTemplateRequest mapping(Map source) { private PutIndexTemplateRequest internalMapping(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(source); MediaType mediaType = builder.contentType(); Objects.requireNonNull(mediaType); diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java index 721d6094f7502..6d7e95d191ba6 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java @@ -38,7 +38,6 @@ import org.opensearch.client.TimedRequest; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -111,7 +110,7 @@ public MediaType mediaType() { */ public PutMappingRequest source(Map mappingSource) { try { - XContentBuilder builder = XContentFactory.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); builder.map(mappingSource); return source(builder); } catch (IOException e) { diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java index 707f4246009aa..e9b297d21bcba 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java @@ -36,7 +36,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -60,7 +59,7 @@ public final void testFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference bytes = toShuffledXContent(clientTestInstance, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, bytes.streamInput()); final S serverInstance = doParseToServerInstance(parser); assertInstances(serverInstance, clientTestInstance); diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java index 5f7df0cd5860c..b5ee31c87b8d4 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java @@ -36,7 +36,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -59,7 +58,7 @@ public final void testFromXContent() throws IOException { final S serverTestInstance = createServerTestInstance(xContentType); final BytesReference bytes = toShuffledXContent(serverTestInstance, xContentType, getParams(), randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java index 03d6e6720ffed..376eb3c5e407c 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java @@ -35,9 +35,9 @@ import org.opensearch.client.AbstractResponseTestCase; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -61,7 +61,7 @@ static class SourceOnlyResponse implements ToXContentObject { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { // this implementation copied from RestGetSourceAction.RestGetSourceResponseListener::buildResponse try (InputStream stream = source.streamInput()) { - builder.rawValue(stream, XContentHelper.xContentType(source)); + builder.rawValue(stream, MediaTypeRegistry.xContentType(source)); } return builder; } diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java index b83bdab899a41..7babb339e176a 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java @@ -39,7 +39,6 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.index.Index; @@ -194,7 +193,7 @@ public final void testBwcFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference bytes = toShuffledXContent(expected, xContentType, getParams(), randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, @@ -215,7 +214,7 @@ public final void testBwcFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference bytes = toShuffledXContent(expected, xContentType, getParams(), randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java index 6f924898096a0..8c83791d94d42 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java @@ -39,10 +39,10 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -262,7 +262,7 @@ private static AliasMetadata randomAliasMetadata(String name) { } static XContentBuilder randomMapping(String type, XContentType xContentType) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); builder.startObject().startObject(type); randomMappingFields(builder, true); diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java index 7193cd3bd97bb..8e3c115c7ba58 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java @@ -34,6 +34,8 @@ import org.opensearch.core.common.io.stream.Writeable; +import java.io.IOException; +import java.io.OutputStream; import java.util.Locale; /** @@ -69,12 +71,20 @@ default String typeWithSubtype() { XContent xContent(); + boolean detectedXContent(final byte[] bytes, int offset, int length); + + boolean detectedXContent(final CharSequence content, final int length); + default String mediaType() { return mediaTypeWithoutParameters(); } String mediaTypeWithoutParameters(); + XContentBuilder contentBuilder() throws IOException; + + XContentBuilder contentBuilder(final OutputStream os) throws IOException; + /** * Accepts a format string, which is most of the time is equivalent to {@link MediaType#subtype()} * and attempts to match the value to an {@link MediaType}. diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java index b81325f6c7c74..064f154602caf 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java @@ -32,8 +32,15 @@ package org.opensearch.core.xcontent; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.spi.MediaTypeProvider; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -54,6 +61,7 @@ public final class MediaTypeRegistry { // Default mediaType singleton private static MediaType DEFAULT_MEDIA_TYPE; + public static final int GUESS_HEADER_LENGTH = 20; // JSON is a core type, so we create a static instance for implementations that require JSON format (e.g., tests) // todo we should explore moving the concrete JSON implementation from the xcontent library to core @@ -118,6 +126,213 @@ public static MediaType fromFormat(String format) { return formatToMediaType.get(format.toLowerCase(Locale.ROOT)); } + /** + * Returns a binary content builder for the provided content type. + */ + public static XContentBuilder contentBuilder(MediaType type) throws IOException { + for (var mediaType : formatToMediaType.values()) { + if (type == mediaType) { + return type.contentBuilder(); + } + } + throw new IllegalArgumentException("No matching content type for " + type); + } + + public static XContentBuilder contentBuilder(MediaType type, OutputStream outputStream) throws IOException { + for (var mediaType : formatToMediaType.values()) { + if (type == mediaType) { + return type.contentBuilder(outputStream); + } + } + throw new IllegalArgumentException("No matching content type for " + type); + } + + /** + * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent} + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContent(final byte[] data, int offset, int length) { + MediaType type = mediaTypeFromBytes(data, offset, length); + if (type == null) { + throw new XContentParseException("Failed to derive xcontent"); + } + return type; + } + + /** + * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContent(byte[] data) { + return xContent(data, 0, data.length); + } + + /** + * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent} + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContent(CharSequence content) { + MediaType type = xContentType(content); + if (type == null) { + throw new XContentParseException("Failed to derive xcontent"); + } + return type; + } + + /** + * Guesses the content type based on the provided char sequence. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContentType(CharSequence content) { + int length = content.length() < GUESS_HEADER_LENGTH ? content.length() : GUESS_HEADER_LENGTH; + if (length == 0) { + return null; + } + for (var mediaType : formatToMediaType.values()) { + if (mediaType.detectedXContent(content, length)) { + return mediaType; + } + } + + // fallback for json + for (int i = 0; i < length; i++) { + char c = content.charAt(i); + if (c == '{') { + return MediaType.fromMediaType("application/json"); + } + if (Character.isWhitespace(c) == false) { + break; + } + } + return null; + } + + /** + * Guesses the content type based on the provided input stream without consuming it. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContentType(InputStream si) throws IOException { + /* + * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content + * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the + * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and + * reset. + */ + if (si.markSupported() == false) { + throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass()); + } + si.mark(Integer.MAX_VALUE); + try { + // scan until we find the first non-whitespace character or the end of the stream + int current; + do { + current = si.read(); + if (current == -1) { + return null; + } + } while (Character.isWhitespace((char) current)); + // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte + final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH]; + firstBytes[0] = (byte) current; + int read = 1; + while (read < GUESS_HEADER_LENGTH) { + final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read); + if (r == -1) { + break; + } + read += r; + } + return mediaTypeFromBytes(firstBytes, 0, read); + } finally { + si.reset(); + } + + } + + /** + * Guesses the content type based on the provided bytes. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContentType(BytesReference bytes) { + if (bytes instanceof BytesArray) { + final BytesArray array = (BytesArray) bytes; + return mediaTypeFromBytes(array.array(), array.offset(), array.length()); + } + try { + final InputStream inputStream = bytes.streamInput(); + assert inputStream.markSupported(); + return xContentType(inputStream); + } catch (IOException e) { + assert false : "Should not happen, we're just reading bytes from memory"; + throw new UncheckedIOException(e); + } + } + + /** + * Guesses the content type based on the provided bytes. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType mediaTypeFromBytes(final byte[] data, int offset, int length) { + int totalLength = data.length; + if (totalLength == 0 || length == 0) { + return null; + } else if ((offset + length) > totalLength) { + return null; + } + for (var mediaType : formatToMediaType.values()) { + if (mediaType.detectedXContent(data, offset, length)) { + return mediaType; + } + } + + // a last chance for JSON + int jsonStart = 0; + // JSON may be preceded by UTF-8 BOM + if (length > 3 && data[offset] == (byte) 0xEF && data[offset + 1] == (byte) 0xBB && data[offset + 2] == (byte) 0xBF) { + jsonStart = 3; + } + + for (int i = jsonStart; i < length; i++) { + byte b = data[offset + i]; + if (b == '{') { + return fromMediaType("application/json"); + } + if (Character.isWhitespace(b) == false) { + break; + } + } + + return null; + } + /** * parsing media type that follows https://tools.ietf.org/html/rfc7231#section-3.1.1.1 * @param headerValue a header value from Accept or Content-Type diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java index 76a2046dd768a..9f423bc9abad3 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java @@ -32,19 +32,14 @@ package org.opensearch.common.xcontent; -import com.fasterxml.jackson.dataformat.cbor.CBORConstants; -import com.fasterxml.jackson.dataformat.smile.SmileConstants; -import org.opensearch.common.xcontent.cbor.CborXContent; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; import org.opensearch.common.xcontent.yaml.YamlXContent; -import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParseException; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -52,13 +47,11 @@ */ public class XContentFactory { - static final int GUESS_HEADER_LENGTH = 20; - /** * Returns a content builder using JSON format ({@link org.opensearch.common.xcontent.XContentType#JSON}. */ public static XContentBuilder jsonBuilder() throws IOException { - return contentBuilder(XContentType.JSON); + return MediaTypeRegistry.contentBuilder(XContentType.JSON); } /** @@ -72,7 +65,7 @@ public static XContentBuilder jsonBuilder(OutputStream os) throws IOException { * Returns a content builder using SMILE format ({@link org.opensearch.common.xcontent.XContentType#SMILE}. */ public static XContentBuilder smileBuilder() throws IOException { - return contentBuilder(XContentType.SMILE); + return MediaTypeRegistry.contentBuilder(XContentType.SMILE); } /** @@ -86,7 +79,7 @@ public static XContentBuilder smileBuilder(OutputStream os) throws IOException { * Returns a content builder using YAML format ({@link org.opensearch.common.xcontent.XContentType#YAML}. */ public static XContentBuilder yamlBuilder() throws IOException { - return contentBuilder(XContentType.YAML); + return MediaTypeRegistry.contentBuilder(XContentType.YAML); } /** @@ -100,271 +93,6 @@ public static XContentBuilder yamlBuilder(OutputStream os) throws IOException { * Returns a content builder using CBOR format ({@link org.opensearch.common.xcontent.XContentType#CBOR}. */ public static XContentBuilder cborBuilder() throws IOException { - return contentBuilder(XContentType.CBOR); - } - - /** - * Constructs a new cbor builder that will output the result into the provided output stream. - */ - public static XContentBuilder cborBuilder(OutputStream os) throws IOException { - return new XContentBuilder(CborXContent.cborXContent, os); - } - - /** - * Constructs a xcontent builder that will output the result into the provided output stream. - */ - public static XContentBuilder contentBuilder(MediaType type, OutputStream outputStream) throws IOException { - if (type == XContentType.JSON) { - return jsonBuilder(outputStream); - } else if (type == XContentType.SMILE) { - return smileBuilder(outputStream); - } else if (type == XContentType.YAML) { - return yamlBuilder(outputStream); - } else if (type == XContentType.CBOR) { - return cborBuilder(outputStream); - } - throw new IllegalArgumentException("No matching content type for " + type); - } - - /** - * Returns a binary content builder for the provided media type. - */ - public static XContentBuilder contentBuilder(MediaType type) throws IOException { - if (type instanceof XContentType) { - return contentBuilder((XContentType) (type)); - } - throw new IllegalArgumentException("Content type [" + type.getClass().getName() + "] not supported"); - } - - /** - * Returns a binary content builder for the provided content type. - */ - public static XContentBuilder contentBuilder(XContentType type) throws IOException { - if (type == XContentType.JSON) { - return JsonXContent.contentBuilder(); - } else if (type == XContentType.SMILE) { - return SmileXContent.contentBuilder(); - } else if (type == XContentType.YAML) { - return YamlXContent.contentBuilder(); - } else if (type == XContentType.CBOR) { - return CborXContent.contentBuilder(); - } - throw new IllegalArgumentException("No matching content type for " + type); - } - - /** - * Returns the {@link XContent} for the provided content type. - */ - public static XContent xContent(MediaType type) { - if (type == null) { - throw new IllegalArgumentException("Cannot get xcontent for unknown type"); - } - return type.xContent(); - } - - /** - * Guesses the content type based on the provided char sequence. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(CharSequence content) { - int length = content.length() < GUESS_HEADER_LENGTH ? content.length() : GUESS_HEADER_LENGTH; - if (length == 0) { - return null; - } - char first = content.charAt(0); - if (first == '{') { - return XContentType.JSON; - } - // Should we throw a failure here? Smile idea is to use it in bytes.... - if (length > 2 - && first == SmileConstants.HEADER_BYTE_1 - && content.charAt(1) == SmileConstants.HEADER_BYTE_2 - && content.charAt(2) == SmileConstants.HEADER_BYTE_3) { - return XContentType.SMILE; - } - if (length > 2 && first == '-' && content.charAt(1) == '-' && content.charAt(2) == '-') { - return XContentType.YAML; - } - - // CBOR is not supported - - for (int i = 0; i < length; i++) { - char c = content.charAt(i); - if (c == '{') { - return XContentType.JSON; - } - if (Character.isWhitespace(c) == false) { - break; - } - } - return null; - } - - /** - * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent} - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContent xContent(CharSequence content) { - XContentType type = xContentType(content); - if (type == null) { - throw new XContentParseException("Failed to derive xcontent"); - } - return xContent(type); - } - - /** - * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContent xContent(byte[] data) { - return xContent(data, 0, data.length); - } - - /** - * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} - * - * @deprecated guessing the content type should not be needed ideally. We should rather know the content type upfront or read it - * from headers. Till we fixed the REST layer to read the Content-Type header, that should be the only place where guessing is needed. - */ - @Deprecated - public static XContent xContent(byte[] data, int offset, int length) { - XContentType type = xContentType(data, offset, length); - if (type == null) { - throw new XContentParseException("Failed to derive xcontent"); - } - return xContent(type); - } - - /** - * Guesses the content type based on the provided input stream without consuming it. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(InputStream si) throws IOException { - /* - * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content - * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the - * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and - * reset. - */ - if (si.markSupported() == false) { - throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass()); - } - si.mark(Integer.MAX_VALUE); - try { - // scan until we find the first non-whitespace character or the end of the stream - int current; - do { - current = si.read(); - if (current == -1) { - return null; - } - } while (Character.isWhitespace((char) current)); - // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte - final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH]; - firstBytes[0] = (byte) current; - int read = 1; - while (read < GUESS_HEADER_LENGTH) { - final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read); - if (r == -1) { - break; - } - read += r; - } - return xContentType(firstBytes, 0, read); - } finally { - si.reset(); - } - - } - - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(byte[] bytes) { - return xContentType(bytes, 0, bytes.length); - } - - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(byte[] bytes, int offset, int length) { - int totalLength = bytes.length; - if (totalLength == 0 || length == 0) { - return null; - } else if ((offset + length) > totalLength) { - return null; - } - byte first = bytes[offset]; - if (first == '{') { - return XContentType.JSON; - } - if (length > 2 - && first == SmileConstants.HEADER_BYTE_1 - && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2 - && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3) { - return XContentType.SMILE; - } - if (length > 2 && first == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-') { - return XContentType.YAML; - } - // CBOR logic similar to CBORFactory#hasCBORFormat - if (first == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) { - return XContentType.CBOR; - } - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first) && length > 2) { - // Actually, specific "self-describe tag" is a very good indicator - if (first == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) { - return XContentType.CBOR; - } - } - // for small objects, some encoders just encode as major type object, we can safely - // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, first)) { - return XContentType.CBOR; - } - - int jsonStart = 0; - // JSON may be preceded by UTF-8 BOM - if (length > 3 && first == (byte) 0xEF && bytes[offset + 1] == (byte) 0xBB && bytes[offset + 2] == (byte) 0xBF) { - jsonStart = 3; - } - - // a last chance for JSON - for (int i = jsonStart; i < length; i++) { - byte b = bytes[offset + i]; - if (b == '{') { - return XContentType.JSON; - } - if (Character.isWhitespace(b) == false) { - break; - } - } - return null; + return MediaTypeRegistry.contentBuilder(XContentType.CBOR); } } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java index d418e25ba8292..7026dbd5e78c3 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java @@ -32,6 +32,8 @@ package org.opensearch.common.xcontent; +import com.fasterxml.jackson.dataformat.cbor.CBORConstants; +import com.fasterxml.jackson.dataformat.smile.SmileConstants; import org.opensearch.common.xcontent.cbor.CborXContent; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; @@ -39,8 +41,10 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContent; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; +import java.io.OutputStream; /** * The content type of {@link XContent}. @@ -70,6 +74,26 @@ public String subtype() { public XContent xContent() { return JsonXContent.jsonXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + return bytes[offset] == '{'; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return content.charAt(0) == '{'; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return JsonXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(JsonXContent.jsonXContent, os); + } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -89,6 +113,32 @@ public String subtype() { public XContent xContent() { return SmileXContent.smileXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + return length > 2 + && bytes[offset] == SmileConstants.HEADER_BYTE_1 + && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2 + && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return length > 2 + && content.charAt(0) == SmileConstants.HEADER_BYTE_1 + && content.charAt(1) == SmileConstants.HEADER_BYTE_2 + && content.charAt(2) == SmileConstants.HEADER_BYTE_3; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return SmileXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(SmileXContent.smileXContent, os); + } }, /** * A YAML based content type. @@ -108,6 +158,26 @@ public String subtype() { public XContent xContent() { return YamlXContent.yamlXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + return length > 2 && bytes[offset] == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-'; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return length > 2 && content.charAt(0) == '-' && content.charAt(1) == '-' && content.charAt(2) == '-'; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return YamlXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(YamlXContent.yamlXContent, os); + } }, /** * A CBOR based content type. @@ -127,6 +197,41 @@ public String subtype() { public XContent xContent() { return CborXContent.cborXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + // CBOR logic similar to CBORFactory#hasCBORFormat + if (bytes[offset] == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) { + return true; + } + if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, bytes[offset]) && length > 2) { + // Actually, specific "self-describe tag" is a very good indicator + if (bytes[offset] == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) { + return true; + } + } + // for small objects, some encoders just encode as major type object, we can safely + // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort + if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, bytes[offset])) { + return true; + } + return false; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return false; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return CborXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(CborXContent.cborXContent, os); + } }; private int index; diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java index 9164b5216f9a5..3d4bbba9cc50c 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java @@ -43,9 +43,9 @@ import com.fasterxml.jackson.core.util.JsonGeneratorDelegate; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -339,7 +339,7 @@ public void writeRawField(String name, InputStream content) throws IOException { // needed for the XContentFactory.xContentType call content = new BufferedInputStream(content); } - XContentType contentType = XContentFactory.xContentType(content); + MediaType contentType = MediaTypeRegistry.xContentType(content); if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } @@ -354,7 +354,7 @@ public void writeRawField(String name, InputStream content, MediaType mediaType) if (mayWriteRawData(mediaType) == false) { // EMPTY is safe here because we never call namedObject when writing raw data try ( - XContentParser parser = XContentFactory.xContent(mediaType) + XContentParser parser = mediaType.xContent() // It's okay to pass the throwing deprecation handler // because we should not be writing raw fields when // generating JSON diff --git a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java index b189c8ed8905e..87598aa28ce8f 100644 --- a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java @@ -35,7 +35,6 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -68,7 +67,7 @@ final class UserAgentParser { private void init(InputStream regexStream) throws IOException { // EMPTY is safe here because we don't use namedObject - XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML) + XContentParser yamlParser = XContentType.YAML.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, regexStream); XContentParser.Token token = yamlParser.nextToken(); diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java index 6e56ecf3950bb..da67a0d2dd13a 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java @@ -40,8 +40,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; @@ -104,11 +104,11 @@ public static SearchTemplateResponse fromXContent(XContentParser parser) throws if (contentAsMap.containsKey(TEMPLATE_OUTPUT_FIELD.getPreferredName())) { Object source = contentAsMap.get(TEMPLATE_OUTPUT_FIELD.getPreferredName()); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).value(source); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).value(source); searchTemplateResponse.setSource(BytesReference.bytes(builder)); } else { MediaType contentType = parser.contentType(); - XContentBuilder builder = XContentFactory.contentBuilder(contentType).map(contentAsMap); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType).map(contentAsMap); XContentParser searchResponseParser = contentType.xContent() .createParser(parser.getXContentRegistry(), parser.getDeprecationHandler(), BytesReference.bytes(builder).streamInput()); diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java index bbda8d15d9d41..f6b0cc4eecf9a 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java @@ -42,7 +42,6 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.action.search.RestSearchAction; @@ -132,8 +131,7 @@ static SearchRequest convert( } try ( - XContentParser parser = XContentFactory.xContent(XContentType.JSON) - .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source) + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source) ) { SearchSourceBuilder builder = SearchSourceBuilder.searchSource(); builder.parseXContent(parser, false); diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java index 664e87da0a7d8..e3669ef67ff36 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java @@ -33,9 +33,9 @@ package org.opensearch.script.mustache; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -101,7 +101,7 @@ public void testToXContentWithInlineTemplate() throws IOException { request.setScriptParams(scriptParams); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedRequest = MediaTypeRegistry.contentBuilder(contentType) .startObject() .field("source", "{\"query\": { \"match\" : { \"{{my_field}}\" : \"{{my_value}}\" } } }") .startObject("params") @@ -112,7 +112,7 @@ public void testToXContentWithInlineTemplate() throws IOException { .field("profile", true) .endObject(); - XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); + XContentBuilder actualRequest = MediaTypeRegistry.contentBuilder(contentType); request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType); @@ -131,7 +131,7 @@ public void testToXContentWithStoredTemplate() throws IOException { request.setScriptParams(params); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedRequest = MediaTypeRegistry.contentBuilder(contentType) .startObject() .field("id", "match_template") .startObject("params") @@ -142,7 +142,7 @@ public void testToXContentWithStoredTemplate() throws IOException { .field("profile", false) .endObject(); - XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); + XContentBuilder actualRequest = MediaTypeRegistry.contentBuilder(contentType); request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType); diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java index 804a72561c10e..fd0a4e9612a8f 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java @@ -36,6 +36,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -164,7 +165,7 @@ public void testSourceToXContent() throws IOException { response.setSource(BytesReference.bytes(source)); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType) .startObject() .startObject("template_output") .startObject("query") @@ -175,7 +176,7 @@ public void testSourceToXContent() throws IOException { .endObject() .endObject(); - XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType); + XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType); response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType); @@ -210,7 +211,7 @@ public void testSearchResponseToXContent() throws IOException { response.setResponse(searchResponse); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType) .startObject() .field("took", 0) .field("timed_out", false) @@ -235,7 +236,7 @@ public void testSearchResponseToXContent() throws IOException { .endObject() .endObject(); - XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType); + XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType); response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType); diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java index d5b61d5c5a517..3500086564719 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java @@ -71,6 +71,8 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -123,7 +125,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder documents; - private final XContentType documentXContentType; + private final MediaType documentXContentType; private final String indexedDocumentIndex; private final String indexedDocumentId; @@ -150,7 +152,7 @@ public PercolateQueryBuilder(String field, BytesReference document, XContentType * @param documents The binary blob containing document to percolate * @param documentXContentType The content type of the binary blob containing the document to percolate */ - public PercolateQueryBuilder(String field, List documents, XContentType documentXContentType) { + public PercolateQueryBuilder(String field, List documents, MediaType documentXContentType) { if (field == null) { throw new IllegalArgumentException("[field] is a required argument"); } @@ -252,7 +254,11 @@ protected PercolateQueryBuilder(String field, Supplier documentS } documents = in.readList(StreamInput::readBytesReference); if (documents.isEmpty() == false) { - documentXContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + documentXContentType = in.readMediaType(); + } else { + documentXContentType = in.readEnum(XContentType.class); + } } else { documentXContentType = null; } @@ -298,7 +304,11 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBytesReference(document); } if (documents.isEmpty() == false) { - out.writeEnum(documentXContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + documentXContentType.writeTo(out); + } else { + out.writeEnum((XContentType) documentXContentType); + } } } @@ -432,7 +442,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) { PercolateQueryBuilder rewritten = new PercolateQueryBuilder( field, Collections.singletonList(source), - XContentHelper.xContentType(source) + MediaTypeRegistry.xContentType(source) ); if (name != null) { rewritten.setName(name); @@ -560,7 +570,7 @@ public List getDocuments() { } // pkg-private for testing - XContentType getXContentType() { + MediaType getXContentType() { return documentXContentType; } diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java index ea001de0ee7c6..d97d5a3a7dcd5 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -36,9 +36,9 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -274,7 +274,7 @@ public static DiscountedCumulativeGain createTestItem() { public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGain testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java index 014f52faa9d57..32c7edc845e32 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -166,7 +166,7 @@ public static ExpectedReciprocalRank createTestItem() { public void testXContentRoundtrip() throws IOException { ExpectedReciprocalRank testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java index 3df79acfa6ce1..d34e6ba8bd72e 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -183,7 +183,7 @@ public void testNoResults() throws Exception { public void testXContentRoundtrip() throws IOException { MeanReciprocalRank testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java index 3317a2d2f00f1..41340cfd1003d 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -199,7 +199,7 @@ public static PrecisionAtK createTestItem() { public void testXContentRoundtrip() throws IOException { PrecisionAtK testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java index 7c0590566bba9..d027026a6b317 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java @@ -44,9 +44,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -177,7 +177,7 @@ public void testToXContent() throws IOException { Collections.singletonMap("coffee_query", coffeeQueryQuality), Collections.singletonMap("beer_query", new ParsingException(new XContentLocation(0, 0), "someMsg")) ); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); String xContent = BytesReference.bytes(response.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString(); assertEquals( ("{" diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java index 988784b6e57a3..01f5a3a12da01 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java @@ -34,9 +34,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -57,7 +57,7 @@ public static RatedDocument createRatedDocument() { public void testXContentParsing() throws IOException { RatedDocument testItem = createRatedDocument(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { RatedDocument parsedItem = RatedDocument.fromXContent(itemParser); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java index ac2401f30e6f0..6e99e31a2b819 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java @@ -35,10 +35,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -134,7 +134,7 @@ public static RatedRequest createTestItem(boolean forceRequest) { public void testXContentRoundtrip() throws IOException { RatedRequest testItem = createTestItem(randomBoolean()); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java index 85e024f6bb1e9..37f778fbc5059 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -185,7 +185,7 @@ public static RecallAtK createTestItem() { public void testXContentRoundtrip() throws IOException { RecallAtK testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java index 6f36fafd852ef..4b1e416ac63ea 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java @@ -37,8 +37,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.search.RestSearchAction; @@ -106,7 +106,7 @@ private XContentParser extractRequestSpecificFields(RestRequest restRequest, Map } try ( XContentParser parser = restRequest.contentOrSourceParamParser(); - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()) + XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType()) ) { Map body = parser.map(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java index e554a3be20528..fe62f5902cbba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java @@ -35,8 +35,8 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.WrapperQueryBuilder; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; @@ -59,7 +59,7 @@ public void testWrapperQueryIsRewritten() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); BytesReference bytesReference; - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.startObject(); { builder.startObject("terms"); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 5da3f2eb01260..3bb552b80084d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -387,7 +388,7 @@ public CreateSnapshotRequest settings(String source, MediaType mediaType) { */ public CreateSnapshotRequest settings(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(source); settings(Strings.toString(builder), builder.contentType()); } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java index 9118cdd56babd..2f304668df01c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java @@ -40,6 +40,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -127,7 +128,7 @@ public Alias filter(Map filter) { return this; } try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(filter); this.filter = Strings.toString(builder); return this; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java index 142cbe6a0ab0b..6ed9648502ee3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -48,6 +48,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.ToXContent; @@ -428,7 +429,7 @@ public AliasActions filter(Map filter) { return this; } try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(filter); this.filter = org.opensearch.common.Strings.toString(builder); return this; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java index 465a44556c081..2588dfd718e71 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -49,6 +49,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.index.Index; @@ -298,7 +299,7 @@ public PutMappingRequest source(XContentBuilder mappingBuilder) { */ public PutMappingRequest source(Map mappingSource) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(mappingSource); return source(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequest.java b/server/src/main/java/org/opensearch/action/index/IndexRequest.java index d686f0b460634..96d27917d5164 100644 --- a/server/src/main/java/org/opensearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/opensearch/action/index/IndexRequest.java @@ -54,11 +54,11 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; @@ -378,7 +378,7 @@ public IndexRequest source(Map source) throws OpenSearchGenerationExc */ public IndexRequest source(Map source, MediaType contentType) throws OpenSearchGenerationException { try { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType); builder.map(source); return source(builder); } catch (IOException e) { @@ -434,7 +434,7 @@ public IndexRequest source(MediaType mediaType, Object... source) { ); } try { - XContentBuilder builder = XContentFactory.contentBuilder(mediaType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(mediaType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java index 71200b05d70ad..57cc4698cefce 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java @@ -49,8 +49,8 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; @@ -306,7 +306,7 @@ public TermVectorsRequest doc(XContentBuilder documentBuilder) { */ @Deprecated public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { - return this.doc(doc, generateRandomId, XContentHelper.xContentType(doc)); + return this.doc(doc, generateRandomId, MediaTypeRegistry.xContentType(doc)); } /** diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java index 44454630ff24d..86ebc0d9b69d6 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java @@ -49,11 +49,11 @@ import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -107,12 +107,12 @@ public class UpdateRequest extends InstanceShardOperationRequest ); PARSER.declareBoolean(UpdateRequest::scriptedUpsert, SCRIPTED_UPSERT_FIELD); PARSER.declareObject((request, builder) -> request.safeUpsertRequest().source(builder), (parser, context) -> { - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType()); builder.copyCurrentStructure(parser); return builder; }, UPSERT_FIELD); PARSER.declareObject((request, builder) -> request.safeDoc().source(builder), (parser, context) -> { - XContentBuilder docBuilder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder docBuilder = MediaTypeRegistry.contentBuilder(parser.contentType()); docBuilder.copyCurrentStructure(parser); return docBuilder; }, DOC_FIELD); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java index 44f6f2d7313a3..27ecca0358bd8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java @@ -45,6 +45,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -307,7 +308,7 @@ public Builder filter(String filter) { this.filter = null; return this; } - return filter(XContentHelper.convertToMap(XContentFactory.xContent(filter), filter, true)); + return filter(XContentHelper.convertToMap(MediaTypeRegistry.xContent(filter).xContent(), filter, true)); } public Builder filter(Map filter) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java index 5a019804f5eac..9b9b91a07a5cf 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java @@ -36,10 +36,10 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -88,7 +88,7 @@ public void validateAliasStandalone(Alias alias) { validateAliasStandalone(alias.name(), alias.indexRouting()); if (Strings.hasLength(alias.filter())) { try { - XContentHelper.convertToMap(XContentFactory.xContent(alias.filter()), alias.filter(), false); + XContentHelper.convertToMap(MediaTypeRegistry.xContent(alias.filter()).xContent(), alias.filter(), false); } catch (Exception e) { throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e); } @@ -134,7 +134,8 @@ public void validateAliasFilter( ) { assert queryShardContext != null; try ( - XContentParser parser = XContentFactory.xContent(filter) + XContentParser parser = MediaTypeRegistry.xContent(filter) + .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter) ) { validateAliasFilter(parser, queryShardContext); @@ -158,7 +159,7 @@ public void validateAliasFilter( try ( InputStream inputStream = filter.streamInput(); - XContentParser parser = XContentFactory.xContentType(inputStream) + XContentParser parser = MediaTypeRegistry.xContentType(inputStream) .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter.streamInput()) ) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 1ba38daa40566..bcf9cbd0efef8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -53,9 +53,9 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -1296,7 +1296,7 @@ public Builder putMapping(String source) throws IOException { putMapping( new MappingMetadata( MapperService.SINGLE_MAPPING_NAME, - XContentHelper.convertToMap(XContentFactory.xContent(source), source, true) + XContentHelper.convertToMap(MediaTypeRegistry.xContent(source).xContent(), source, true) ) ); return this; diff --git a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java index 62ec933fe5f37..ee2d5a650fc70 100644 --- a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java +++ b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java @@ -34,8 +34,8 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.compress.NotXContentException; +import org.opensearch.core.xcontent.MediaTypeRegistry; import java.io.IOException; import java.util.Objects; @@ -67,14 +67,14 @@ public static Compressor compressor(BytesReference bytes) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem - assert XContentHelper.xContentType(bytes) == null; + assert MediaTypeRegistry.xContentType(bytes) == null; return DEFLATE_COMPRESSOR; } else if (ZSTD_COMPRESSOR.isCompressed(bytes)) { - assert XContentHelper.xContentType(bytes) == null; + assert MediaTypeRegistry.xContentType(bytes) == null; return ZSTD_COMPRESSOR; } - if (XContentHelper.xContentType(bytes) == null) { + if (MediaTypeRegistry.xContentType(bytes) == null) { throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index a7649a1cd22c5..10a619d833add 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -48,7 +48,7 @@ import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.settings.SecureString; @@ -1081,7 +1081,7 @@ private void processLegacyLists(Map map) { */ public Builder loadFromMap(Map map) { // TODO: do this without a serialization round-trip - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.map(map); return loadFromSource(Strings.toString(builder), builder.contentType()); } catch (IOException e) { @@ -1094,7 +1094,7 @@ public Builder loadFromMap(Map map) { */ public Builder loadFromSource(String source, MediaType xContentType) { try ( - XContentParser parser = XContentFactory.xContent(xContentType) + XContentParser parser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source) ) { this.put(fromXContent(parser, true, true)); @@ -1127,7 +1127,7 @@ public Builder loadFromStream(String resourceName, InputStream is, boolean accep } // fromXContent doesn't use named xcontent or deprecation. try ( - XContentParser parser = XContentFactory.xContent(xContentType) + XContentParser parser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, is) ) { if (parser.currentToken() == null) { diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java index ae1fb0724fd9e..a0903cbdd9f61 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java @@ -41,6 +41,7 @@ import org.opensearch.common.compress.CompressorFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContent.Params; @@ -52,7 +53,6 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -86,14 +86,14 @@ public static XContentParser createParser( if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - final XContentType contentType = XContentFactory.xContentType(compressedInput); - return XContentFactory.xContent(contentType).createParser(xContentRegistry, deprecationHandler, compressedInput); + final MediaType contentType = MediaTypeRegistry.xContentType(compressedInput); + return contentType.xContent().createParser(xContentRegistry, deprecationHandler, compressedInput); } catch (Exception e) { if (compressedInput != null) compressedInput.close(); throw e; } } else { - return XContentFactory.xContent(xContentType(bytes)).createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); + return MediaTypeRegistry.xContentType(bytes).xContent().createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); } } @@ -115,7 +115,7 @@ public static XContentParser createParser( if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - return XContentFactory.xContent(mediaType).createParser(xContentRegistry, deprecationHandler, compressedInput); + return mediaType.xContent().createParser(xContentRegistry, deprecationHandler, compressedInput); } catch (Exception e) { if (compressedInput != null) compressedInput.close(); throw e; @@ -179,20 +179,14 @@ public static Tuple> convertToMap( final byte[] raw = arr.array(); final int offset = arr.offset(); final int length = arr.length(); - contentType = xContentType != null ? xContentType : XContentFactory.xContentType(raw, offset, length); - return new Tuple<>( - Objects.requireNonNull(contentType), - convertToMap(XContentFactory.xContent(contentType), raw, offset, length, ordered) - ); + contentType = xContentType != null ? xContentType : MediaTypeRegistry.mediaTypeFromBytes(raw, offset, length); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(contentType.xContent(), raw, offset, length, ordered)); } else { input = bytes.streamInput(); } try (InputStream stream = input) { - contentType = xContentType != null ? xContentType : XContentFactory.xContentType(stream); - return new Tuple<>( - Objects.requireNonNull(contentType), - convertToMap(XContentFactory.xContent(contentType), stream, ordered) - ); + contentType = xContentType != null ? xContentType : MediaTypeRegistry.xContentType(stream); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(contentType.xContent(), stream, ordered)); } } catch (IOException e) { throw new OpenSearchParseException("Failed to parse content to map", e); @@ -266,7 +260,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson) t @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException { - return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes.toBytesRef().bytes)); + return convertToJson(bytes, reformatJson, prettyPrint, MediaTypeRegistry.xContent(bytes.toBytesRef().bytes)); } public static String convertToJson(BytesReference bytes, boolean reformatJson, MediaType xContentType) throws IOException { @@ -311,7 +305,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, b if (bytes instanceof BytesArray) { final BytesArray array = (BytesArray) bytes; try ( - XContentParser parser = XContentFactory.xContent(mediaType) + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, @@ -325,7 +319,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, b } else { try ( InputStream stream = bytes.streamInput(); - XContentParser parser = XContentFactory.xContent(mediaType) + XContentParser parser = mediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream) ) { return toJsonString(prettyPrint, parser); @@ -545,29 +539,6 @@ public static BytesReference toXContent(ToXContent toXContent, MediaType mediaTy } } - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(BytesReference bytes) { - if (bytes instanceof BytesArray) { - final BytesArray array = (BytesArray) bytes; - return XContentFactory.xContentType(array.array(), array.offset(), array.length()); - } - try { - final InputStream inputStream = bytes.streamInput(); - assert inputStream.markSupported(); - return XContentFactory.xContentType(inputStream); - } catch (IOException e) { - assert false : "Should not happen, we're just reading bytes from memory"; - throw new UncheckedIOException(e); - } - } - /** * Returns the contents of an object as an unparsed BytesReference * diff --git a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java index 9943f04c964e1..2a419031b2c5a 100644 --- a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java @@ -49,12 +49,12 @@ import org.opensearch.common.lucene.store.IndexOutputOutputStream; import org.opensearch.common.lucene.store.InputStreamIndexInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.common.util.io.IOUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -288,7 +288,7 @@ private long write(final T state, boolean cleanup, final Path... locations) thro } protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException { - return XContentFactory.contentBuilder(type, stream); + return MediaTypeRegistry.contentBuilder(type, stream); } /** @@ -321,7 +321,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { try ( - XContentParser parser = XContentFactory.xContent(FORMAT) + XContentParser parser = FORMAT.xContent() .createParser( namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java index caddf92150cff..c01f51eb7188a 100644 --- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java @@ -67,7 +67,6 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.RecyclingBytesStreamOutput; import org.opensearch.common.io.Streams; import org.opensearch.common.logging.Loggers; @@ -76,14 +75,15 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; -import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.util.ByteArray; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -448,7 +448,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw final SetOnce builderReference = new SetOnce<>(); consumeFromType(searcher, GLOBAL_TYPE_NAME, bytes -> { final Metadata metadata = Metadata.Builder.fromXContent( - XContentFactory.xContent(XContentType.SMILE) + XContentType.SMILE.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length) ); logger.trace("found global metadata with last-accepted term [{}]", metadata.coordinationMetadata().term()); @@ -468,7 +468,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw final Set indexUUIDs = new HashSet<>(); consumeFromType(searcher, INDEX_TYPE_NAME, bytes -> { final IndexMetadata indexMetadata = IndexMetadata.fromXContent( - XContentFactory.xContent(XContentType.SMILE) + XContentType.SMILE.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length) ); logger.trace("found index metadata for {}", indexMetadata.getIndex()); @@ -921,7 +921,7 @@ private Document makeDocument(String typeName, ToXContent metadata, DocumentBuff try (RecyclingBytesStreamOutput streamOutput = documentBuffer.streamOutput()) { try ( - XContentBuilder xContentBuilder = XContentFactory.contentBuilder( + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder( XContentType.SMILE, Streams.flushOnCloseStream(streamOutput) ) diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java index b3715e097322d..8f81e704ef2d4 100644 --- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java @@ -51,10 +51,10 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.util.set.Sets; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.engine.Engine; @@ -299,7 +299,7 @@ private GetResult innerGetLoadFromStoredFields( shardId.getIndexName(), id, source, - XContentHelper.xContentType(source), + MediaTypeRegistry.xContentType(source), fieldVisitor.routing() ); ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse); @@ -378,7 +378,7 @@ private GetResult innerGetLoadFromStoredFields( sourceAsMap = typeMapTuple.v2(); sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes()); try { - source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap)); + source = BytesReference.bytes(MediaTypeRegistry.contentBuilder(sourceContentType).map(sourceAsMap)); } catch (IOException e) { throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e); } @@ -406,7 +406,7 @@ private GetResult innerGetLoadFromStoredFields( sourceAsMap = typeMapTuple.v2(); sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes()); try { - source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap)); + source = BytesReference.bytes(MediaTypeRegistry.contentBuilder(sourceContentType).map(sourceAsMap)); } catch (IOException e) { throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e); } diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java index ad467d045b7e4..deda9883bc07b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java @@ -39,15 +39,15 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Nullable; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.CollectionUtils; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; @@ -228,7 +228,7 @@ public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nul Map filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); MediaType actualContentType = mapTuple.v1(); - XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(actualContentType, bStream).map(filteredSource); builder.close(); return bStream.bytes(); } else { diff --git a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java index aab9adecb9a62..b0fc62ad8ec71 100644 --- a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java @@ -34,15 +34,15 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; @@ -175,7 +175,8 @@ protected boolean doEquals(WrapperQueryBuilder other) { @Override protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException { try ( - XContentParser qSourceParser = XContentFactory.xContent(source) + XContentParser qSourceParser = MediaTypeRegistry.xContent(source) + .xContent() .createParser(context.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, source) ) { diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java index 40b15eace2bad..a4c4aa9493288 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java @@ -49,10 +49,10 @@ import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.IndexGeoPointFieldData; @@ -230,7 +230,8 @@ protected ScoreFunction doToFunction(QueryShardContext context) throws IOExcepti // EMPTY is safe because parseVariable doesn't use namedObject try ( InputStream stream = functionBytes.streamInput(); - XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(functionBytes)) + XContentParser parser = MediaTypeRegistry.xContentType(functionBytes) + .xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream) ) { scoreFunction = parseVariable(fieldName, parser, context, multiValueMode); diff --git a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java index b3ed0079ff4c2..2f03eb66bd71e 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java @@ -49,8 +49,8 @@ import org.opensearch.common.document.DocumentField; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.search.SearchHit; import org.opensearch.threadpool.ThreadPool; @@ -210,7 +210,7 @@ public BytesReference getSource() { @Override public MediaType getMediaType() { - return XContentHelper.xContentType(source); + return MediaTypeRegistry.xContentType(source); } @Override diff --git a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java index f6b4793f3b87a..8a9a346a5344a 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java @@ -44,10 +44,10 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.VersionType; import org.opensearch.index.query.QueryBuilder; @@ -356,7 +356,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws request.getSearchRequest().indices(indices); } request.setRemoteInfo(buildRemoteInfo(source)); - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType()); builder.map(source); try ( InputStream stream = BytesReference.bytes(builder).streamInput(); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index e43b9773cc1e0..e09a218ccf83b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -98,11 +98,11 @@ import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.gateway.WriteStateException; import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; @@ -2171,7 +2171,7 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o shardId.getIndexName(), index.id(), index.source(), - XContentHelper.xContentType(index.source()), + MediaTypeRegistry.xContentType(index.source()), index.routing() ) ); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b574ffd1006c0..f4841a8ca9a28 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -82,12 +82,12 @@ import org.opensearch.common.util.iterable.Iterables; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.util.FileSystemUtils; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.env.NodeEnvironment; @@ -1727,7 +1727,7 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set filterParser = bytes -> { try ( InputStream inputStream = bytes.streamInput(); - XContentParser parser = XContentFactory.xContentType(inputStream) + XContentParser parser = MediaTypeRegistry.xContentType(inputStream) .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, inputStream) ) { diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java index cb3d779ece4a9..a138bfc8a7044 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -43,20 +43,20 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.CheckedFunction; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.Compressor; import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.lucene.store.IndexOutputOutputStream; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.gateway.CorruptStateException; import org.opensearch.snapshots.SnapshotInfo; @@ -185,7 +185,7 @@ public void close() throws IOException { // in order to write the footer we need to prevent closing the actual index input. } }; - XContentBuilder builder = XContentFactory.contentBuilder( + XContentBuilder builder = MediaTypeRegistry.contentBuilder( XContentType.SMILE, compressor.threadLocalOutputStream(indexOutputOutputStream) ) diff --git a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java index 32499b1fc155b..11a116e8c858d 100644 --- a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java @@ -38,7 +38,6 @@ import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import java.io.IOException; import java.io.OutputStream; @@ -145,12 +144,7 @@ public XContentBuilder newBuilder(@Nullable MediaType requestContentType, @Nulla } OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); - XContentBuilder builder = new XContentBuilder( - XContentFactory.xContent(responseContentType), - unclosableOutputStream, - includes, - excludes - ); + XContentBuilder builder = new XContentBuilder(responseContentType.xContent(), unclosableOutputStream, includes, excludes); if (pretty) { builder.prettyPrint().lfAtEnd(); } diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java index b4abe449e027f..ce5e12d6fd163 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java @@ -38,8 +38,8 @@ import org.opensearch.action.get.GetResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; @@ -111,7 +111,7 @@ public RestResponse buildResponse(final GetResponse response) throws Exception { final XContentBuilder builder = channel.newBuilder(request.getMediaType(), false); final BytesReference source = response.getSourceInternal(); try (InputStream stream = source.streamInput()) { - builder.rawValue(stream, XContentHelper.xContentType(source)); + builder.rawValue(stream, MediaTypeRegistry.xContentType(source)); } return new BytesRestResponse(OK, builder); } diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java index 6f3c3f471b47c..1276bc824c05b 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -38,11 +38,11 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -111,7 +111,7 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder { // Copy the current structure. We will parse, once the mapping is provided - XContentBuilder builder = XContentFactory.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); builder.copyCurrentStructure(p); v.contextBytes = BytesReference.bytes(builder); p.skipChildren(); @@ -216,7 +216,7 @@ public CompletionSuggestionBuilder regex(String regex, RegexOptions regexOptions public CompletionSuggestionBuilder contexts(Map> queryContexts) { Objects.requireNonNull(queryContexts, "contexts must not be null"); try { - XContentBuilder contentBuilder = XContentFactory.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); contentBuilder.startObject(); for (Map.Entry> contextEntry : queryContexts.entrySet()) { contentBuilder.startArray(contextEntry.getKey()); diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java index 4b1c43bf7a317..6992af62b6626 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java @@ -44,7 +44,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.core.common.text.Text; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.ParsedQuery; @@ -160,7 +160,8 @@ public Suggestion> innerExecute( QueryShardContext shardContext = suggestion.getShardContext(); final String querySource = scriptFactory.newInstance(vars).execute(); try ( - XContentParser parser = XContentFactory.xContent(querySource) + XContentParser parser = MediaTypeRegistry.xContent(querySource) + .xContent() .createParser(shardContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, querySource) ) { QueryBuilder innerQueryBuilder = AbstractQueryBuilder.parseInnerQueryBuilder(parser); diff --git a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java index 28a09fd80b408..041a5d32b027c 100644 --- a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java +++ b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java @@ -36,8 +36,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -76,7 +76,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { try (InputStream stream = status.streamInput()) { - return builder.rawValue(stream, XContentHelper.xContentType(status)); + return builder.rawValue(stream, MediaTypeRegistry.xContentType(status)); } } diff --git a/server/src/main/java/org/opensearch/tasks/TaskResult.java b/server/src/main/java/org/opensearch/tasks/TaskResult.java index b6006636da2e9..34efbe3ec572b 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResult.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResult.java @@ -42,10 +42,10 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.InstantiatingObjectParser; import org.opensearch.common.xcontent.ObjectParserHelper; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -238,7 +238,7 @@ public int hashCode() { } private static BytesReference toXContent(Exception error) throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { builder.startObject(); OpenSearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, error); builder.endObject(); diff --git a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java index 1feb115cb585a..01be4eaaaf732 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java @@ -54,9 +54,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.Streams; import org.opensearch.threadpool.ThreadPool; @@ -169,7 +169,7 @@ private int getTaskResultMappingVersion(IndexMetadata metadata) { private void doStoreResult(TaskResult taskResult, ActionListener listener) { IndexRequestBuilder index = client.prepareIndex(TASK_INDEX).setId(taskResult.getTask().getTaskId().toString()); - try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS); index.setSource(builder); } catch (IOException e) { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 6885278b08413..c463f7a8f4b8c 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -56,9 +56,9 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.tasks.Task; @@ -761,7 +761,7 @@ public void testTasksToXContentGrouping() throws Exception { } private Map serialize(ListTasksResponse response, boolean byParents) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject(); if (byParents) { DiscoveryNodes nodes = testNodes[0].clusterService.state().nodes(); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index e92096f139d17..c9b57d0c41b45 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -46,10 +46,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.NetworkModule; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.admin.cluster.RestClusterRerouteAction; @@ -212,7 +212,7 @@ private ClusterRerouteRequest roundTripThroughRestRequest(ClusterRerouteRequest private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException { Map params = new HashMap<>(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); boolean hasBody = false; if (randomBoolean()) { builder.prettyPrint(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java index 265f02304ce00..2c4462b964d65 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java @@ -37,9 +37,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -195,7 +195,7 @@ public void testParseAdd() throws IOException { if (filter == null || filter.isEmpty()) { assertNull(action.filter()); } else { - assertEquals(Strings.toString(XContentFactory.contentBuilder(XContentType.JSON).map(filter)), action.filter()); + assertEquals(Strings.toString(MediaTypeRegistry.contentBuilder(XContentType.JSON).map(filter)), action.filter()); } assertEquals(Objects.toString(searchRouting, null), action.searchRouting()); assertEquals(Objects.toString(indexRouting, null), action.indexRouting()); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java index e4114a622602c..c8a24fd0482cf 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -38,9 +38,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -101,7 +101,7 @@ public void testMappingKeyedByType() throws IOException { CreateIndexRequest request1 = new CreateIndexRequest("foo"); CreateIndexRequest request2 = new CreateIndexRequest("bar"); { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject("properties") .startObject("field1") @@ -117,7 +117,7 @@ public void testMappingKeyedByType() throws IOException { .endObject() .endObject(); request1.mapping(builder); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject(MapperService.SINGLE_MAPPING_NAME) .startObject("properties") @@ -140,7 +140,7 @@ public void testMappingKeyedByType() throws IOException { } public void testSettingsType() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject().startArray("settings").endArray().endObject(); CreateIndexRequest parsedCreateIndexRequest = new CreateIndexRequest(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java index 4b23a68f8b383..9de9e81ee9cae 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -43,6 +43,7 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; @@ -179,7 +180,7 @@ public void testSerialize() throws Exception { public void testUnknownFields() throws IOException { final RolloverRequest request = new RolloverRequest(); XContentType xContentType = randomFrom(XContentType.values()); - final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + final XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); builder.startObject(); { builder.startObject("conditions"); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index dbca883506b0d..f069ff3bf7ff2 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -35,6 +35,7 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; @@ -76,7 +77,7 @@ public void testMappingKeyedByType() throws IOException { PutIndexTemplateRequest request1 = new PutIndexTemplateRequest("foo"); PutIndexTemplateRequest request2 = new PutIndexTemplateRequest("bar"); { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject("properties") .startObject("field1") @@ -92,7 +93,7 @@ public void testMappingKeyedByType() throws IOException { .endObject() .endObject(); request1.mapping(builder); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject("properties") .startObject("field1") diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java index c674be8dbba38..2f2c9b2be9e50 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java @@ -39,14 +39,14 @@ import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Requests; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.script.Script; import org.opensearch.test.OpenSearchTestCase; @@ -340,7 +340,7 @@ public void testSmileIsSupported() throws IOException { XContentType xContentType = XContentType.SMILE; BytesReference data; try (BytesStreamOutput out = new BytesStreamOutput()) { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("index"); builder.field("_index", "index"); @@ -349,7 +349,7 @@ public void testSmileIsSupported() throws IOException { builder.endObject(); } out.write(xContentType.xContent().streamSeparator()); - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.field("field", "value"); builder.endObject(); @@ -375,7 +375,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException XContentType xContentType = XContentType.SMILE; BytesReference data; try (BytesStreamOutput out = new BytesStreamOutput()) { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("update"); builder.field("_index", "index"); @@ -386,7 +386,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException builder.endObject(); } out.write(xContentType.xContent().streamSeparator()); - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("doc").endObject(); Map values = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java index 866e13c7faf0e..14f2a5c94bcd0 100644 --- a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java +++ b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java @@ -37,9 +37,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.get.GetResult; @@ -114,7 +114,7 @@ public void testToXContent() throws IOException { ); ExplainResponse response = new ExplainResponse(index, id, exist, explanation, getResult); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); response.toXContent(builder, ToXContent.EMPTY_PARAMS); String generatedResponse = BytesReference.bytes(builder).utf8ToString().replaceAll("\\s+", ""); diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 382425ce4e57c..d76ed86ed490e 100644 --- a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -34,9 +34,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.AbstractSerializingTestCase; @@ -125,7 +125,7 @@ protected Predicate getRandomFieldsExcludeFilter() { public void testToXContent() throws IOException { FieldCapabilitiesResponse response = createSimpleResponse(); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); response.toXContent(builder, ToXContent.EMPTY_PARAMS); String generatedResponse = BytesReference.bytes(builder).utf8ToString(); diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java index 5498d685dbb6c..cc1e963bc110d 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java @@ -126,7 +126,7 @@ public void testXContentSerialization() throws IOException { MultiGetRequest expected = createTestInstance(); XContentType xContentType = randomFrom(XContentType.values()); BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false); - try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) { + try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) { MultiGetRequest actual = new MultiGetRequest(); actual.add(null, null, null, null, parser, true); assertThat(parser.nextToken(), nullValue()); diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java index 71ada43bef116..c2a5f2def31ab 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java @@ -33,7 +33,6 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.get.GetResult; @@ -53,7 +52,7 @@ public void testFromXContent() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false); MultiGetResponse parsed; - try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) { + try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) { parsed = MultiGetResponse.fromXContent(parser); assertNull(parser.nextToken()); } diff --git a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java index 0e25091d845b5..fafd34602c310 100644 --- a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java +++ b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java @@ -38,10 +38,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent.MapParams; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -422,7 +422,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException { final boolean allowNoIndices = randomBoolean(); BytesReference xContentBytes; - try (XContentBuilder builder = XContentFactory.contentBuilder(type)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) { builder.startObject(); builder.field("expand_wildcards", "all"); builder.field("ignore_unavailable", ignoreUnavailable); @@ -441,7 +441,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException { assertTrue(fromXContentOptions.expandWildcardsHidden()); assertTrue(fromXContentOptions.expandWildcardsOpen()); - try (XContentBuilder builder = XContentFactory.contentBuilder(type)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) { builder.startObject(); builder.field("expand_wildcards", "none"); builder.field("ignore_unavailable", ignoreUnavailable); @@ -461,7 +461,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException { } private BytesReference toXContentBytes(IndicesOptions indicesOptions, XContentType type) throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(type)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) { builder.startObject(); indicesOptions.toXContent(builder, new MapParams(Collections.emptyMap())); builder.endObject(); diff --git a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java index 417a79da68ae1..4968f7740b7c9 100644 --- a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -37,9 +37,9 @@ import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.Writeable.Reader; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -86,7 +86,7 @@ public static void afterClass() throws Exception { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { SB testShape = createTestShapeBuilder(); - XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { contentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java index 274b6477e3fbd..00ab96dab6a5c 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java @@ -49,6 +49,7 @@ import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -865,7 +866,7 @@ public void testBasics() throws IOException { generator.writeEndObject(); } byte[] data = os.toByteArray(); - assertEquals(xcontentType(), XContentFactory.xContentType(data)); + assertEquals(xcontentType(), MediaTypeRegistry.xContent(data)); } public void testMissingEndObject() throws IOException { diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java index 866def49af991..1b7c535a550b0 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java @@ -37,6 +37,7 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; @@ -64,14 +65,14 @@ public void testGuessCbor() throws IOException { } private void testGuessType(XContentType type) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(type); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(type); builder.startObject(); builder.field("field1", "value1"); builder.endObject(); final BytesReference bytes; if (type == XContentType.JSON && randomBoolean()) { - final int length = randomIntBetween(0, 8 * XContentFactory.GUESS_HEADER_LENGTH); + final int length = randomIntBetween(0, 8 * MediaTypeRegistry.GUESS_HEADER_LENGTH); final String content = Strings.toString(builder); final StringBuilder sb = new StringBuilder(length + content.length()); final char[] chars = new char[length]; @@ -82,24 +83,24 @@ private void testGuessType(XContentType type) throws IOException { bytes = BytesReference.bytes(builder); } - assertThat(XContentHelper.xContentType(bytes), equalTo(type)); - assertThat(XContentFactory.xContentType(bytes.streamInput()), equalTo(type)); + assertThat(MediaTypeRegistry.xContentType(bytes), equalTo(type)); + assertThat(MediaTypeRegistry.xContentType(bytes.streamInput()), equalTo(type)); // CBOR is binary, cannot use String if (type != XContentType.CBOR && type != XContentType.SMILE) { - assertThat(XContentFactory.xContentType(Strings.toString(builder)), equalTo(type)); + assertThat(MediaTypeRegistry.xContentType(Strings.toString(builder)), equalTo(type)); } } public void testCBORBasedOnMajorObjectDetection() { // for this {"f "=> 5} perl encoder for example generates: byte[] bytes = new byte[] { (byte) 0xA1, (byte) 0x43, (byte) 0x66, (byte) 6f, (byte) 6f, (byte) 0x5 }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR)); // assertThat(((Number) XContentHelper.convertToMap(bytes, true).v2().get("foo")).intValue(), equalTo(5)); // this if for {"foo" : 5} in python CBOR bytes = new byte[] { (byte) 0xA1, (byte) 0x63, (byte) 0x66, (byte) 0x6f, (byte) 0x6f, (byte) 0x5 }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR)); assertThat(((Number) XContentHelper.convertToMap(new BytesArray(bytes), true).v2().get("foo")).intValue(), equalTo(5)); // also make sure major type check doesn't collide with SMILE and JSON, just in case @@ -111,36 +112,36 @@ public void testCBORBasedOnMajorObjectDetection() { public void testCBORBasedOnMagicHeaderDetection() { byte[] bytes = new byte[] { (byte) 0xd9, (byte) 0xd9, (byte) 0xf7 }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR)); } public void testEmptyStream() throws Exception { ByteArrayInputStream is = new ByteArrayInputStream(new byte[0]); - assertNull(XContentFactory.xContentType(is)); + assertNull(MediaTypeRegistry.xContentType(is)); is = new ByteArrayInputStream(new byte[] { (byte) 1 }); - assertNull(XContentFactory.xContentType(is)); + assertNull(MediaTypeRegistry.xContentType(is)); } public void testInvalidStream() throws Exception { byte[] bytes = new byte[] { (byte) '"' }; - assertNull(XContentFactory.xContentType(bytes)); + assertNull(MediaTypeRegistry.mediaTypeFromBytes(bytes, 0, bytes.length)); bytes = new byte[] { (byte) 'x' }; - assertNull(XContentFactory.xContentType(bytes)); + assertNull(MediaTypeRegistry.mediaTypeFromBytes(bytes, 0, bytes.length)); } public void testJsonFromBytesOptionallyPrecededByUtf8Bom() throws Exception { byte[] bytes = new byte[] { (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); bytes = new byte[] { (byte) 0x20, (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); bytes = new byte[] { (byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); bytes = new byte[] { (byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) 0x20, (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); } } diff --git a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java index 6c97297ce43e7..dc98000d93899 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java @@ -39,13 +39,13 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentOpenSearchExtension; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentGenerator; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentGenerator; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayOutputStream; @@ -68,7 +68,7 @@ public class XContentBuilderTests extends OpenSearchTestCase { public void testPrettyWithLfAtEnd() throws Exception { ByteArrayOutputStream os = new ByteArrayOutputStream(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); + XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os); generator.usePrettyPrint(); generator.usePrintLineFeedAtEnd(); @@ -87,7 +87,7 @@ public void testPrettyWithLfAtEnd() throws Exception { public void testReuseJsonGenerator() throws Exception { ByteArrayOutputStream os = new ByteArrayOutputStream(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); + XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os); generator.writeStartObject(); generator.writeStringField("test", "value"); generator.writeEndObject(); @@ -107,14 +107,14 @@ public void testReuseJsonGenerator() throws Exception { public void testRaw() throws IOException { { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); xContentBuilder.endObject(); assertThat(Strings.toString(xContentBuilder), equalTo("{\"foo\":{\"test\":\"value\"}}")); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -122,7 +122,7 @@ public void testRaw() throws IOException { assertThat(Strings.toString(xContentBuilder), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}")); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -130,7 +130,7 @@ public void testRaw() throws IOException { assertThat(Strings.toString(xContentBuilder), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}")); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -142,7 +142,7 @@ public void testRaw() throws IOException { ); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -157,17 +157,17 @@ public void testRaw() throws IOException { } public void testSimpleGenerator() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", "value").endObject(); assertThat(Strings.toString(builder), equalTo("{\"test\":\"value\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", "value").endObject(); assertThat(Strings.toString(builder), equalTo("{\"test\":\"value\"}")); } public void testOverloadedList() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", Arrays.asList("1", "2")).endObject(); assertThat(Strings.toString(builder), equalTo("{\"test\":[\"1\",\"2\"]}")); } @@ -175,7 +175,7 @@ public void testOverloadedList() throws Exception { public void testWritingBinaryToStream() throws Exception { BytesStreamOutput bos = new BytesStreamOutput(); - XContentGenerator gen = XContentFactory.xContent(XContentType.JSON).createGenerator(bos); + XContentGenerator gen = XContentType.JSON.xContent().createGenerator(bos); gen.writeStartObject(); gen.writeStringField("name", "something"); gen.flush(); @@ -189,7 +189,7 @@ public void testWritingBinaryToStream() throws Exception { } public void testByteConversion() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test_name", (Byte) (byte) 120).endObject(); assertThat(BytesReference.bytes(builder).utf8ToString(), equalTo("{\"test_name\":120}")); } @@ -199,21 +199,21 @@ public void testDateTypesConversion() throws Exception { String expectedDate = XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(date.getTime()); Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT); String expectedCalendar = XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(calendar.getTimeInMillis()); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().timeField("date", date).endObject(); assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("calendar", calendar).endObject(); assertThat(Strings.toString(builder), equalTo("{\"calendar\":\"" + expectedCalendar + "\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); Map map = new HashMap<>(); map.put("date", date); builder.map(map); assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); map = new HashMap<>(); map.put("calendar", calendar); builder.map(map); @@ -221,7 +221,7 @@ public void testDateTypesConversion() throws Exception { } public void testCopyCurrentStructure() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", "test field").startObject("filter").startObject("terms"); // up to 20k random terms @@ -248,7 +248,7 @@ public void testCopyCurrentStructure() throws Exception { } } else if (token == XContentParser.Token.START_OBJECT) { if ("filter".equals(currentFieldName)) { - filterBuilder = XContentFactory.contentBuilder(parser.contentType()); + filterBuilder = MediaTypeRegistry.contentBuilder(parser.contentType()); filterBuilder.copyCurrentStructure(parser); } } @@ -288,10 +288,10 @@ public void testHandlingOfPath_absolute() throws IOException { } private void checkPathSerialization(Path path) throws IOException { - XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); pathBuilder.startObject().field("file", path).endObject(); - XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); stringBuilder.startObject().field("file", path.toString()).endObject(); assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder))); @@ -301,10 +301,10 @@ public void testHandlingOfPath_StringName() throws IOException { Path path = PathUtils.get("path"); String name = new String("file"); - XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); pathBuilder.startObject().field(name, path).endObject(); - XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); stringBuilder.startObject().field(name, path.toString()).endObject(); assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder))); @@ -313,36 +313,36 @@ public void testHandlingOfPath_StringName() throws IOException { public void testHandlingOfCollectionOfPaths() throws IOException { Path path = PathUtils.get("path"); - XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); pathBuilder.startObject().field("file", Arrays.asList(path)).endObject(); - XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); stringBuilder.startObject().field("file", Arrays.asList(path.toString())).endObject(); assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder))); } public void testIndentIsPlatformIndependent() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint(); builder.startObject().field("test", "foo").startObject("foo").field("foobar", "boom").endObject().endObject(); String string = Strings.toString(builder); assertEquals("{\n" + " \"test\" : \"foo\",\n" + " \"foo\" : {\n" + " \"foobar\" : \"boom\"\n" + " }\n" + "}", string); - builder = XContentFactory.contentBuilder(XContentType.YAML).prettyPrint(); + builder = MediaTypeRegistry.contentBuilder(XContentType.YAML).prettyPrint(); builder.startObject().field("test", "foo").startObject("foo").field("foobar", "boom").endObject().endObject(); string = Strings.toString(builder); assertEquals("---\n" + "test: \"foo\"\n" + "foo:\n" + " foobar: \"boom\"\n", string); } public void testRenderGeoPoint() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint(); builder.startObject().field("foo").value(new GeoPoint(1, 2)).endObject(); String string = Strings.toString(builder); assertEquals("{\n" + " \"foo\" : {\n" + " \"lat\" : 1.0,\n" + " \"lon\" : 2.0\n" + " }\n" + "}", string.trim()); } public void testWriteMapWithNullKeys() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); try { builder.map(Collections.singletonMap(null, "test")); fail("write map should have failed"); @@ -352,7 +352,7 @@ public void testWriteMapWithNullKeys() throws IOException { } public void testWriteMapValueWithNullKeys() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); try { builder.map(Collections.singletonMap(null, "test")); fail("write map should have failed"); @@ -362,7 +362,7 @@ public void testWriteMapValueWithNullKeys() throws IOException { } public void testWriteFieldMapWithNullKeys() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); try { builder.startObject(); builder.field("map", Collections.singletonMap(null, "test")); @@ -374,7 +374,7 @@ public void testWriteFieldMapWithNullKeys() throws IOException { public void testMissingEndObject() throws IOException { IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { builder.startObject(); builder.field("foo", true); } @@ -385,7 +385,7 @@ public void testMissingEndObject() throws IOException { public void testMissingEndArray() throws IOException { IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { builder.startObject(); builder.startArray("foo"); builder.value(0); diff --git a/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java b/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java index e304798a0ff0c..89add6ea78722 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java @@ -33,7 +33,6 @@ package org.opensearch.common.xcontent.cbor; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -48,10 +47,10 @@ public class JsonVsCborTests extends OpenSearchTestCase { public void testCompareParsingTokens() throws IOException { BytesStreamOutput xsonOs = new BytesStreamOutput(); - XContentGenerator xsonGen = XContentFactory.xContent(XContentType.CBOR).createGenerator(xsonOs); + XContentGenerator xsonGen = XContentType.CBOR.xContent().createGenerator(xsonOs); BytesStreamOutput jsonOs = new BytesStreamOutput(); - XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs); + XContentGenerator jsonGen = XContentType.JSON.xContent().createGenerator(jsonOs); xsonGen.writeStartObject(); jsonGen.writeStartObject(); diff --git a/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java b/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java index 4f26bdc7e9c26..a96031f2a1dad 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java @@ -33,7 +33,6 @@ package org.opensearch.common.xcontent.smile; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -48,10 +47,10 @@ public class JsonVsSmileTests extends OpenSearchTestCase { public void testCompareParsingTokens() throws IOException { BytesStreamOutput xsonOs = new BytesStreamOutput(); - XContentGenerator xsonGen = XContentFactory.xContent(XContentType.SMILE).createGenerator(xsonOs); + XContentGenerator xsonGen = XContentType.SMILE.xContent().createGenerator(xsonOs); BytesStreamOutput jsonOs = new BytesStreamOutput(); - XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs); + XContentGenerator jsonGen = XContentType.JSON.xContent().createGenerator(jsonOs); xsonGen.writeStartObject(); jsonGen.writeStartObject(); diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index c318dd89f573c..0a8df4406b5cd 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -39,7 +39,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.AbstractFilteringTestCase; @@ -89,7 +88,7 @@ static void assertXContentBuilderAsString(final XContentBuilder expected, final } static void assertXContentBuilderAsBytes(final XContentBuilder expected, final XContentBuilder actual) { - XContent xContent = XContentFactory.xContent(actual.contentType()); + XContent xContent = actual.contentType().xContent(); try ( XContentParser jsonParser = xContent.createParser( NamedXContentRegistry.EMPTY, diff --git a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java index 518e5e880f4f8..9e467dca17409 100644 --- a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java @@ -38,7 +38,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -75,7 +75,7 @@ public void testNoFormat() throws Exception { ) ); - assertThat(XContentFactory.xContentType(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON)); documentMapper = parser.parse("type", new CompressedXContent(mapping)); doc = documentMapper.parse( @@ -87,7 +87,7 @@ public void testNoFormat() throws Exception { ) ); - assertThat(XContentHelper.xContentType(doc.source()), equalTo(XContentType.SMILE)); + assertThat(MediaTypeRegistry.xContentType(doc.source()), equalTo(XContentType.SMILE)); } public void testIncludes() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java index 92efd19732ef0..54afde41daa5b 100644 --- a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java @@ -37,8 +37,8 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -181,7 +181,7 @@ public void testIllegalArguments() { // https://github.com/elastic/elasticsearch/issues/7240 public void testEmptyBooleanQuery() throws Exception { - XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); contentBuilder.startObject().startObject("bool").endObject().endObject(); try (XContentParser xParser = createParser(contentBuilder)) { Query parsedQuery = parseQuery(xParser).toQuery(createShardContext()); diff --git a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java index cb75889a0aa24..5e6226561582e 100644 --- a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java @@ -34,9 +34,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -133,7 +133,7 @@ public void testSerializationOrder() throws Exception { public void testFromAndToXContent() throws Exception { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { InnerHitBuilder innerHit = randomInnerHits(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS); // fields is printed out as an object but parsed into a List where order matters, we disable shuffling XContentBuilder shuffled = shuffleXContent(builder, "fields"); diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java index 0f957517d48e1..200688f35352c 100644 --- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java +++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java @@ -36,9 +36,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -73,7 +73,7 @@ public void testToFromXContent() throws IOException { ); ByteSizeValue size = new ByteSizeValue(Math.abs(randomLong())); BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint(); BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS); byte[] xcontent = BytesReference.toBytes(BytesReference.bytes(shuffleXContent(builder))); @@ -126,7 +126,7 @@ public void testInvalidFieldsInFromXContent() throws IOException { fail("shouldn't be here"); } - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject(); builder.field(FileInfo.NAME, name); builder.field(FileInfo.PHYSICAL_NAME, physicalName); diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java index ccb89ec3d1547..eb2006124662a 100644 --- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java +++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java @@ -10,9 +10,9 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -177,7 +177,7 @@ public void testFromXContentInvalid() throws IOException { fail("shouldn't be here"); } - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject(); builder.field(RemoteStoreShardShallowCopySnapshot.VERSION, version); builder.field(RemoteStoreShardShallowCopySnapshot.NAME, snapshot); diff --git a/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java b/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java index fdaf6d145da64..82e6e1c9ff450 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -68,7 +68,7 @@ public void testFromXContent() throws IOException { map.put(pipeline.getId(), pipeline); map.put(pipeline2.getId(), pipeline2); IngestMetadata ingestMetadata = new IngestMetadata(map); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.prettyPrint(); builder.startObject(); ingestMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java b/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java index 7fae4d5bb23e3..56e2ae56fdc71 100644 --- a/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java +++ b/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java @@ -54,7 +54,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.persistent.PersistentTasksCustomMetadata.Assignment; @@ -204,7 +203,7 @@ public void testSerializationContext() throws Exception { BytesReference shuffled = toShuffledXContent(testInstance, xContentType, params, false); PersistentTasksCustomMetadata newInstance; - try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) { + try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) { newInstance = doParseInstance(parser); } assertNotSame(newInstance, testInstance); diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java index 9739419406851..1fbc628ed7f2c 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -33,8 +33,8 @@ package org.opensearch.rest.action.admin.indices; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestResponse; import org.opensearch.test.OpenSearchTestCase; @@ -60,7 +60,7 @@ public class RestGetAliasesActionTests extends OpenSearchTestCase { // }' public void testBareRequest() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata foobarAliasMetadata = AliasMetadata.builder("foobar").build(); final AliasMetadata fooAliasMetadata = AliasMetadata.builder("foo").build(); @@ -72,7 +72,7 @@ public void testBareRequest() throws Exception { } public void testSimpleAliasWildcardMatchingNothing() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse( true, @@ -86,7 +86,7 @@ public void testSimpleAliasWildcardMatchingNothing() throws Exception { } public void testMultipleAliasWildcardsSomeMatching() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata aliasMetadata = AliasMetadata.builder("foobar").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); @@ -102,7 +102,7 @@ public void testMultipleAliasWildcardsSomeMatching() throws Exception { } public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse( true, @@ -116,7 +116,7 @@ public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { } public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata aliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); @@ -132,7 +132,7 @@ public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { } public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata aliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); @@ -153,7 +153,7 @@ public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws E } public void testAliasWildcardsExcludeExplicitMissing() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse( true, diff --git a/server/src/test/java/org/opensearch/script/ScriptTests.java b/server/src/test/java/org/opensearch/script/ScriptTests.java index 47b4557c3e5e1..bb74bd2e5f6b0 100644 --- a/server/src/test/java/org/opensearch/script/ScriptTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptTests.java @@ -37,6 +37,7 @@ import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -58,7 +59,7 @@ public class ScriptTests extends OpenSearchTestCase { public void testScriptParsing() throws IOException { Script expectedScript = createScript(); - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS); try (XContentParser parser = createParser(builder)) { Script actualScript = Script.parse(parser); @@ -103,7 +104,7 @@ private Script createScript() throws IOException { public void testParse() throws IOException { Script expectedScript = createScript(); - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS); try (XContentParser xParser = createParser(builder)) { Settings settings = Settings.fromXContent(xParser); diff --git a/server/src/test/java/org/opensearch/script/StoredScriptTests.java b/server/src/test/java/org/opensearch/script/StoredScriptTests.java index c16c4f93e6582..96c2b19bf204f 100644 --- a/server/src/test/java/org/opensearch/script/StoredScriptTests.java +++ b/server/src/test/java/org/opensearch/script/StoredScriptTests.java @@ -37,8 +37,8 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.AbstractSerializingTestCase; @@ -73,7 +73,7 @@ public void testInvalidDelete() { public void testSourceParsing() throws Exception { // simple script value string - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().startObject("script").field("lang", "lang").field("source", "code").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -83,7 +83,7 @@ public void testSourceParsing() throws Exception { } // complex template using script as the field name - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject() .startObject("script") .field("lang", "mustache") @@ -94,7 +94,7 @@ public void testSourceParsing() throws Exception { .endObject(); String code; - try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) { + try (XContentBuilder cb = MediaTypeRegistry.contentBuilder(builder.contentType())) { code = Strings.toString(cb.startObject().field("query", "code").endObject()); } @@ -109,7 +109,7 @@ public void testSourceParsing() throws Exception { } // complex script with script object - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").field("source", "code").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -119,7 +119,7 @@ public void testSourceParsing() throws Exception { } // complex script using "code" backcompat - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -130,7 +130,7 @@ public void testSourceParsing() throws Exception { assertWarnings("Deprecated field [code] used, expected [source] instead"); // complex script with script object and empty options - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject() .field("script") .startObject() @@ -149,7 +149,7 @@ public void testSourceParsing() throws Exception { } // complex script with embedded template - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { Strings.toString( builder.startObject() .field("script") @@ -165,7 +165,7 @@ public void testSourceParsing() throws Exception { ); String code; - try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) { + try (XContentBuilder cb = MediaTypeRegistry.contentBuilder(builder.contentType())) { code = Strings.toString(cb.startObject().field("query", "code").endObject()); } @@ -182,7 +182,7 @@ public void testSourceParsing() throws Exception { public void testSourceParsingErrors() throws Exception { // check for missing lang parameter when parsing a script - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("source", "code").endObject().endObject(); IllegalArgumentException iae = expectThrows( @@ -193,7 +193,7 @@ public void testSourceParsingErrors() throws Exception { } // check for missing source parameter when parsing a script - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").endObject().endObject(); IllegalArgumentException iae = expectThrows( @@ -204,7 +204,7 @@ public void testSourceParsingErrors() throws Exception { } // check for illegal options parameter when parsing a script - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject() .field("script") .startObject() @@ -224,7 +224,7 @@ public void testSourceParsingErrors() throws Exception { } // check for unsupported template context - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("template", "code").endObject(); ParsingException pEx = expectThrows( ParsingException.class, @@ -238,7 +238,7 @@ public void testSourceParsingErrors() throws Exception { } public void testEmptyTemplateDeprecations() throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -248,7 +248,7 @@ public void testEmptyTemplateDeprecations() throws IOException { assertWarnings("empty templates should no longer be used"); } - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "mustache").field("source", "").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); diff --git a/server/src/test/java/org/opensearch/search/NestedIdentityTests.java b/server/src/test/java/org/opensearch/search/NestedIdentityTests.java index 341570247ffa9..17b1da2ba3a8c 100644 --- a/server/src/test/java/org/opensearch/search/NestedIdentityTests.java +++ b/server/src/test/java/org/opensearch/search/NestedIdentityTests.java @@ -35,9 +35,9 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -66,7 +66,7 @@ public static NestedIdentity createTestItem(int depth) { public void testFromXContent() throws IOException { NestedIdentity nestedIdentity = createTestItem(randomInt(3)); XContentType xcontentType = randomFrom(XContentType.values()); - XContentBuilder builder = XContentFactory.contentBuilder(xcontentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xcontentType); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java index 1f29a17deeaa8..3a527ec7455d3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java @@ -37,10 +37,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -239,7 +239,7 @@ public void testInvalidType() throws Exception { public void testRewriteAggregation() throws Exception { XContentType xContentType = randomFrom(XContentType.values()); BytesReference bytesReference; - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.startObject(); { builder.startObject("terms"); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java index 3a8829d5a6433..136210a636224 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.BoolQueryBuilder; @@ -106,7 +106,7 @@ public void testFiltersSortedByKey() { } public void testOtherBucket() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").startObject().startObject("term").field("field", "foo").endObject().endObject().endArray(); builder.endObject(); @@ -116,7 +116,7 @@ public void testOtherBucket() throws IOException { // The other bucket is disabled by default assertFalse(filters.otherBucket()); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").startObject().startObject("term").field("field", "foo").endObject().endObject().endArray(); builder.field("other_bucket_key", "some_key"); @@ -128,7 +128,7 @@ public void testOtherBucket() throws IOException { // but setting a key enables it automatically assertTrue(filters.otherBucket()); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").startObject().startObject("term").field("field", "foo").endObject().endObject().endArray(); builder.field("other_bucket", false); @@ -207,7 +207,7 @@ public void testRewritePreservesOtherBucket() throws IOException { public void testEmptyFilters() throws IOException { { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").endArray(); // unkeyed array builder.endObject(); @@ -221,7 +221,7 @@ public void testEmptyFilters() throws IOException { } { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startObject("filters").endObject(); // keyed object builder.endObject(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java index 1423fd6771674..d0b3d84bc78f9 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java @@ -37,9 +37,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.fielddata.AbstractSortedSetDocValues; @@ -218,7 +218,7 @@ public void testRegexExclude() throws IOException { // Serializes/deserializes an IncludeExclude statement with a single clause private IncludeExclude serialize(IncludeExclude incExc, ParseField field) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -264,7 +264,7 @@ public void testRegexIncludeAndExclude() throws IOException { // Serializes/deserializes the IncludeExclude statement with include AND // exclude clauses private IncludeExclude serializeMixedRegex(IncludeExclude incExc) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java index 461812077eba9..51ff64321acfa 100644 --- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java @@ -39,9 +39,9 @@ import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -72,7 +72,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase { public void testFromXContent() throws IOException { SearchSourceBuilder testSearchSourceBuilder = createSearchSourceBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -425,7 +425,7 @@ public void testToXContent() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = BytesReference.bytes(builder); Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); @@ -434,7 +434,7 @@ public void testToXContent() throws IOException { { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(RandomQueryBuilder.createQuery(random())); - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = BytesReference.bytes(builder); Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); @@ -448,7 +448,7 @@ public void testToXContentWithPointInTime() throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); TimeValue keepAlive = randomBoolean() ? TimeValue.timeValueHours(1) : null; searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder("id").setKeepAlive(keepAlive)); - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference bytes = BytesReference.bytes(builder); Map sourceAsMap = XContentHelper.convertToMap(bytes, false, xContentType).v2(); diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index b1dfd836f7461..6d31986cabdc4 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -41,10 +41,10 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -137,7 +137,7 @@ public void testEqualsAndHashcode() throws IOException { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { HighlightBuilder highlightBuilder = randomHighlighterBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -547,7 +547,7 @@ public void testOrderSerialization() throws Exception { } protected static XContentBuilder toXContent(HighlightBuilder highlight, XContentType contentType) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java index a66c7de6ced74..3fb2718f5ccc3 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java @@ -36,9 +36,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.text.Text; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -82,7 +82,7 @@ public void testReplaceUnicodeControlCharacters() { public void testFromXContent() throws IOException { HighlightField highlightField = createTestItem(); XContentType xcontentType = randomFrom(XContentType.values()); - XContentBuilder builder = XContentFactory.contentBuilder(xcontentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xcontentType); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java index 1f632251cf588..4913ee92aa79b 100644 --- a/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java @@ -44,6 +44,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -220,7 +221,7 @@ public QueryBuilder aliasFilter(IndexMetadata indexMetadata, String... aliasName return ShardSearchRequest.parseAliasFilter(bytes -> { try ( InputStream inputStream = bytes.streamInput(); - XContentParser parser = XContentFactory.xContentType(inputStream) + XContentParser parser = MediaTypeRegistry.xContentType(inputStream) .xContent() .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream) ) { diff --git a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java index aa7bcf4d63e3d..199ab7d76ec99 100644 --- a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java @@ -40,11 +40,11 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -130,7 +130,7 @@ private RescorerBuilder copy(RescorerBuilder original) throws IOException public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { RescorerBuilder rescoreBuilder = randomRescoreBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java index 47579e31dc7c2..811d3d2784595 100644 --- a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.text.Text; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; @@ -190,7 +191,7 @@ public void testEqualsAndHashcode() throws Exception { public void testFromXContent() throws Exception { for (int runs = 0; runs < 20; runs++) { SearchAfterBuilder searchAfterBuilder = randomJsonSearchFromBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -216,7 +217,7 @@ public void testFromXContentIllegalType() throws Exception { if (type == XContentType.JSON || type == XContentType.YAML) { continue; } - XContentBuilder xContent = XContentFactory.contentBuilder(type); + XContentBuilder xContent = MediaTypeRegistry.contentBuilder(type); xContent.startObject().startArray("search_after").value(new BigDecimal("9223372036854776003.3")).endArray().endObject(); try (XContentParser parser = createParser(xContent)) { parser.nextToken(); diff --git a/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java index 28d96c916bc41..fbbc667cbcb42 100644 --- a/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java @@ -56,9 +56,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; @@ -208,7 +208,7 @@ public void testEqualsAndHashcode() throws Exception { public void testFromXContent() throws Exception { SliceBuilder sliceBuilder = randomSliceBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java index e8d713d15b337..02f32faa6ffd9 100644 --- a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java @@ -40,10 +40,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.env.Environment; @@ -131,7 +131,7 @@ public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { T testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java index bf37e822bfc3c..f8f2564dacbcb 100644 --- a/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java @@ -34,10 +34,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.ConstantScoreQueryBuilder; @@ -83,7 +83,7 @@ protected NamedXContentRegistry xContentRegistry() { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { NestedSortBuilder testItem = createRandomNestedSort(3); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); XContentBuilder shuffled = shuffleXContent(builder); try (XContentParser parser = createParser(shuffled)) { diff --git a/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java index eee13ec56d841..1ac97fded27ce 100644 --- a/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -39,10 +39,10 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.index.Index; @@ -146,7 +146,7 @@ public void testEqualsAndHashcode() { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { SB suggestionBuilder = randomTestBuilder(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { xContentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java index f8996c528cf32..f2d1acc35f356 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java @@ -34,10 +34,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.SearchModule; @@ -82,7 +82,7 @@ public static void afterClass() { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { SuggestBuilder suggestBuilder = randomSuggestBuilder(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { xContentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java index 8484861999c3f..d0e42f8b91e2a 100644 --- a/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java @@ -47,9 +47,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -140,7 +140,7 @@ private static DirectCandidateGeneratorBuilder mutate(DirectCandidateGeneratorBu public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { DirectCandidateGeneratorBuilder generator = randomCandidateGenerator(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java b/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java index a4adf03cff316..9b807ee35e4bf 100644 --- a/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -45,9 +45,9 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.BytesRefs; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.SearchModule; @@ -101,7 +101,7 @@ public static void afterClass() throws Exception { */ public void testFromXContent() throws IOException { SmoothingModel testModel = createTestModel(); - XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { contentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/tasks/TaskResultTests.java b/server/src/test/java/org/opensearch/tasks/TaskResultTests.java index f27a920d812e9..fac7a0538090a 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResultTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResultTests.java @@ -36,9 +36,9 @@ import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -120,7 +120,7 @@ private XContentBuilder addRandomUnknownFields(XContentBuilder builder) throws I map.put("unknown_field" + i, Collections.singletonMap("inner", randomAlphaOfLength(20))); } } - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(parser.contentType()); return xContentBuilder.map(map); } } diff --git a/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java index 3c20f22e392b8..8d951c603425d 100644 --- a/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java @@ -35,8 +35,8 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -96,7 +96,7 @@ public static Settings randomIndexSettings() { * Creates a random mapping */ public static XContentBuilder randomMapping() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); randomMappingFields(builder, true); diff --git a/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java index 6a715599a8e1a..9e4e59d9a4d15 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java @@ -33,8 +33,8 @@ package org.opensearch.index.engine; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.analysis.AnalysisRegistry; @@ -137,7 +137,13 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O final String indexName = mapperService.index().getName(); final Engine.Index engineIndex = IndexShard.prepareIndex( docMapper(MapperService.SINGLE_MAPPING_NAME), - new SourceToParse(indexName, index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()), + new SourceToParse( + indexName, + index.id(), + index.source(), + MediaTypeRegistry.xContentType(index.source()), + index.routing() + ), index.seqNo(), index.primaryTerm(), index.version(), diff --git a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java index e7b1dd2bde1ea..5480c4629ebf6 100644 --- a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java @@ -353,7 +353,7 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } jsonBuilder.endArray(); jsonBuilder.endObject(); - XContentParser parser = XContentFactory.xContent(XContentType.JSON) + XContentParser parser = XContentType.JSON.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java index 89d705c8fd1e8..b7b5fa091810d 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java @@ -36,9 +36,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -67,7 +67,7 @@ public abstract class BaseAggregationTestCase convertToMap(ToXContent part) throws IOExcepti } public static BytesReference convertToXContent(Map map, XContentType xContentType) throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.map(map); return BytesReference.bytes(builder); } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 635dc49ff5166..3045b3fe66edc 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -43,8 +43,8 @@ import org.opensearch.client.NodeSelector; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -185,7 +185,7 @@ private MediaType getContentType(Map headers, XContentType[] sup private BytesRef bodyAsBytesRef(Map bodyAsMap, MediaType mediaType) throws IOException { Map finalBodyAsMap = stash.replaceStashedValues(bodyAsMap); - try (XContentBuilder builder = XContentFactory.contentBuilder(mediaType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(mediaType)) { return BytesReference.bytes(builder.map(finalBodyAsMap)).toBytesRef(); } } diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java index a6482ed09b253..ca07e39719e23 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java @@ -31,10 +31,9 @@ package org.opensearch.test.rest.yaml; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -56,7 +55,7 @@ public class ObjectPathTests extends OpenSearchTestCase { private static XContentBuilder randomXContentBuilder() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); - return XContentBuilder.builder(XContentFactory.xContent(xContentType)); + return XContentBuilder.builder(xContentType.xContent()); } public void testEvaluateObjectPathEscape() throws Exception { @@ -342,7 +341,7 @@ public void testEvaluateArrayAsRoot() throws Exception { xContentBuilder.endObject(); xContentBuilder.endArray(); ObjectPath objectPath = ObjectPath.createFromXContent( - XContentFactory.xContent(xContentBuilder.contentType()), + xContentBuilder.contentType().xContent(), BytesReference.bytes(xContentBuilder) ); Object object = objectPath.evaluate(""); From 883559c8b985521dcf11a326f2ee3ae6f8bb4864 Mon Sep 17 00:00:00 2001 From: Poojita Raj Date: Fri, 28 Jul 2023 13:34:47 -0700 Subject: [PATCH 185/186] Unmute test SegmentReplicationIndexShardTests#testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit (#8931) Signed-off-by: Poojita Raj --- .../index/shard/SegmentReplicationIndexShardTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index cc4fa6f28bafc..9107606326150 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -887,7 +887,6 @@ public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitRefresh() throws testNRTReplicaWithRemoteStorePromotedAsPrimary(true, false); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8817") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(true, true); } From a3aab67ee86bf171a7eb480f0933e0b955fbf4f3 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 28 Jul 2023 17:06:42 -0700 Subject: [PATCH 186/186] Add restore level safeguards to prevent file cache oversubscription (#8606) Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 1 + .../restore/RestoreSnapshotRequest.java | 2 +- .../cluster/routing/RoutingTable.java | 10 ++ .../decider/DiskThresholdDecider.java | 6 +- .../common/settings/ClusterSettings.java | 2 + .../store/remote/filecache/FileCache.java | 17 ++- .../main/java/org/opensearch/node/Node.java | 5 +- .../opensearch/snapshots/RestoreService.java | 83 ++++++++-- .../restore/RestoreSnapshotRequestTests.java | 4 + .../cluster/routing/RoutingTableTests.java | 43 ++++++ .../decider/DiskThresholdDeciderTests.java | 2 + .../snapshots/SnapshotResiliencyTests.java | 143 +++++++++++++++++- .../opensearch/test/OpenSearchTestCase.java | 2 + 13 files changed, 300 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f20b3b8d7b51..861674753861a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) - Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) - [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195) +- Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 840564a4bd7a2..7a142e70305ae 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -517,7 +517,7 @@ public String snapshotUuid() { /** * Sets the storage type for this request. */ - RestoreSnapshotRequest storageType(StorageType storageType) { + public RestoreSnapshotRequest storageType(StorageType storageType) { this.storageType = storageType; return this; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 7934649a6d3eb..d6a67bc714689 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -295,6 +295,16 @@ public ShardsIterator allShardsIncludingRelocationTargets(String[] indices) { return allShardsSatisfyingPredicate(indices, shardRouting -> true, true); } + /** + * All the shards on the node which match the predicate + * @param predicate condition to match + * @return iterator over shards matching the predicate + */ + public ShardsIterator allShardsSatisfyingPredicate(Predicate predicate) { + String[] indices = indicesRouting.keySet().toArray(new String[0]); + return allShardsSatisfyingPredicate(indices, predicate, false); + } + private ShardsIterator allShardsSatisfyingPredicate( String[] indices, Predicate predicate, diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 4b69c05807ae4..61b96184abcc4 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -68,7 +68,7 @@ import static org.opensearch.cluster.routing.RoutingPool.getShardPool; import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING; import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; -import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO; +import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING; /** * The {@link DiskThresholdDecider} checks that the node a shard is potentially @@ -199,8 +199,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing final FileCacheStats fileCacheStats = clusterInfo.getNodeFileCacheStats().getOrDefault(node.nodeId(), null); final long nodeCacheSize = fileCacheStats != null ? fileCacheStats.getTotal().getBytes() : 0; final long totalNodeRemoteShardSize = currentNodeRemoteShardSize + shardSize; - - if (totalNodeRemoteShardSize > DATA_TO_FILE_CACHE_SIZE_RATIO * nodeCacheSize) { + final double dataToFileCacheSizeRatio = DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.get(allocation.metadata().settings()); + if (dataToFileCacheSizeRatio > 0.0f && totalNodeRemoteShardSize > dataToFileCacheSizeRatio * nodeCacheSize) { return allocation.decision( Decision.NO, NAME, diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 46a43842451d9..9da9e1b14d307 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -44,6 +44,7 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.search.backpressure.settings.NodeDuressSettings; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; import org.opensearch.search.backpressure.settings.SearchShardTaskSettings; @@ -643,6 +644,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Settings related to Searchable Snapshots Node.NODE_SEARCH_CACHE_SIZE_SETTING, + FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING, // Settings related to Remote Refresh Segment Pressure RemoteRefreshSegmentPressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED, diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java index 3d23b4d22538c..47b891fdb8d21 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java @@ -11,6 +11,7 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.common.breaker.CircuitBreaker; import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.common.settings.Setting; import org.opensearch.index.store.remote.utils.cache.CacheUsage; import org.opensearch.index.store.remote.utils.cache.RefCountedCache; import org.opensearch.index.store.remote.utils.cache.SegmentedCache; @@ -49,8 +50,20 @@ public class FileCache implements RefCountedCache { private final CircuitBreaker circuitBreaker; - // TODO: Convert the constant into an integer setting - public static final int DATA_TO_FILE_CACHE_SIZE_RATIO = 5; + /** + * Defines a limit of how much total remote data can be referenced as a ratio of the size of the disk reserved for + * the file cache. For example, if 100GB disk space is configured for use as a file cache and the + * remote_data_ratio of 5 is defined, then a total of 500GB of remote data can be loaded as searchable snapshots. + * This is designed to be a safeguard to prevent oversubscribing a cluster. + * Specify a value of zero for no limit, which is the default for compatibility reasons. + */ + public static final Setting DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING = Setting.doubleSetting( + "cluster.filecache.remote_data_ratio", + 0.0, + 0.0, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); public FileCache(SegmentedCache cache, CircuitBreaker circuitBreaker) { this.theCache = cache; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index fe630dfe27e3a..0bc824c5a0704 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -941,8 +941,9 @@ protected Node( clusterModule.getAllocationService(), metadataCreateIndexService, metadataIndexUpgradeService, - clusterService.getClusterSettings(), - shardLimitValidator + shardLimitValidator, + indicesService, + clusterInfoService::getClusterInfo ); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor( diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index d7e89172c5837..119e632928cc7 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -41,6 +41,7 @@ import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterInfo; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateApplier; import org.opensearch.cluster.ClusterStateTaskConfig; @@ -69,6 +70,7 @@ import org.opensearch.cluster.routing.RoutingChangesObserver; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterManagerTaskKeys; @@ -87,6 +89,9 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.index.store.remote.filecache.FileCacheStats; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; @@ -104,6 +109,7 @@ import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Collections.unmodifiableSet; @@ -119,6 +125,8 @@ import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; import static org.opensearch.common.util.set.Sets.newHashSet; import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION; +import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING; +import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING; import static org.opensearch.snapshots.SnapshotUtils.filterIndices; /** @@ -177,6 +185,10 @@ public class RestoreService implements ClusterStateApplier { private final ClusterSettings clusterSettings; + private final IndicesService indicesService; + + private final Supplier clusterInfoSupplier; + private final ClusterManagerTaskThrottler.ThrottlingKey restoreSnapshotTaskKey; private static final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor(); @@ -187,8 +199,9 @@ public RestoreService( AllocationService allocationService, MetadataCreateIndexService createIndexService, MetadataIndexUpgradeService metadataIndexUpgradeService, - ClusterSettings clusterSettings, - ShardLimitValidator shardLimitValidator + ShardLimitValidator shardLimitValidator, + IndicesService indicesService, + Supplier clusterInfoSupplier ) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; @@ -200,6 +213,8 @@ public RestoreService( } this.clusterSettings = clusterService.getClusterSettings(); this.shardLimitValidator = shardLimitValidator; + this.indicesService = indicesService; + this.clusterInfoSupplier = clusterInfoSupplier; // Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction. restoreSnapshotTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.RESTORE_SNAPSHOT_KEY, true); @@ -415,7 +430,6 @@ public ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey( @Override public ClusterState execute(ClusterState currentState) { - RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY); // Check if the snapshot to restore is currently being deleted SnapshotDeletionsInProgress deletionsInProgress = currentState.custom( SnapshotDeletionsInProgress.TYPE, @@ -436,7 +450,9 @@ public ClusterState execute(ClusterState currentState) { ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable()); final Map shards; + final boolean isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(request.storageType().toString()); Set aliases = new HashSet<>(); + long totalRestorableRemoteIndexesSize = 0; if (indices.isEmpty() == false) { // We have some indices to restore @@ -447,17 +463,14 @@ public ClusterState execute(ClusterState currentState) { String index = indexEntry.getValue(); boolean partial = checkPartial(index); + IndexId snapshotIndexId = repositoryData.resolveIndexId(index); IndexMetadata snapshotIndexMetadata = updateIndexSettings( metadata.index(index), request.indexSettings(), request.ignoreIndexSettings() ); - if (IndexModule.Type.REMOTE_SNAPSHOT.match(request.storageType().toString())) { - snapshotIndexMetadata = addSnapshotToIndexSettings( - snapshotIndexMetadata, - snapshot, - repositoryData.resolveIndexId(index) - ); + if (isRemoteSnapshot) { + snapshotIndexMetadata = addSnapshotToIndexSettings(snapshotIndexMetadata, snapshot, snapshotIndexId); } final boolean isSearchableSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match( snapshotIndexMetadata.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) @@ -483,7 +496,7 @@ public ClusterState execute(ClusterState currentState) { restoreUUID, snapshot, snapshotInfo.version(), - repositoryData.resolveIndexId(index), + snapshotIndexId, isSearchableSnapshot, isRemoteStoreShallowCopy, request.getSourceRemoteStoreRepository() @@ -602,6 +615,14 @@ public ClusterState execute(ClusterState currentState) { } for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) { + if (isRemoteSnapshot) { + IndexShardSnapshotStatus.Copy shardStatus = repository.getShardSnapshotStatus( + snapshotInfo.snapshotId(), + snapshotIndexId, + new ShardId(metadata.index(index).getIndex(), shard) + ).asCopy(); + totalRestorableRemoteIndexesSize += shardStatus.getTotalSize(); + } if (!ignoreShards.contains(shard)) { shardsBuilder.put( new ShardId(renamedIndex, shard), @@ -638,6 +659,9 @@ public ClusterState execute(ClusterState currentState) { } checkAliasNameConflicts(indices, aliases); + if (isRemoteSnapshot) { + validateSearchableSnapshotRestorable(totalRestorableRemoteIndexesSize); + } Map updatedDataStreams = new HashMap<>(currentState.metadata().dataStreams()); updatedDataStreams.putAll( @@ -837,6 +861,45 @@ private IndexMetadata updateIndexSettings( return builder.settings(settingsBuilder).build(); } + private void validateSearchableSnapshotRestorable(long totalRestorableRemoteIndexesSize) { + ClusterInfo clusterInfo = clusterInfoSupplier.get(); + double remoteDataToFileCacheRatio = DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.get(clusterService.getSettings()); + Map nodeFileCacheStats = clusterInfo.getNodeFileCacheStats(); + if (nodeFileCacheStats.isEmpty() || remoteDataToFileCacheRatio <= 0.01f) { + return; + } + + long totalNodeFileCacheSize = clusterInfo.getNodeFileCacheStats() + .values() + .stream() + .map(fileCacheStats -> fileCacheStats.getTotal().getBytes()) + .mapToLong(Long::longValue) + .sum(); + + Predicate isRemoteSnapshotShard = shardRouting -> shardRouting.primary() + && indicesService.indexService(shardRouting.index()).getIndexSettings().isRemoteSnapshot(); + + ShardsIterator shardsIterator = clusterService.state() + .routingTable() + .allShardsSatisfyingPredicate(isRemoteSnapshotShard); + + long totalRestoredRemoteIndexesSize = shardsIterator.getShardRoutings() + .stream() + .map(clusterInfo::getShardSize) + .mapToLong(Long::longValue) + .sum(); + + if (totalRestoredRemoteIndexesSize + totalRestorableRemoteIndexesSize > remoteDataToFileCacheRatio + * totalNodeFileCacheSize) { + throw new SnapshotRestoreException( + snapshot, + "Size of the indexes to be restored exceeds the file cache bounds. Increase the file cache capacity on the cluster nodes using " + + NODE_SEARCH_CACHE_SIZE_SETTING.getKey() + + " setting." + ); + } + } + @Override public void onFailure(String source, Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index 5c20b3b262730..82b2cfb2e3e51 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -112,6 +112,10 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { instance.snapshotUuid(randomBoolean() ? null : randomAlphaOfLength(10)); } + instance.storageType( + randomBoolean() ? RestoreSnapshotRequest.StorageType.LOCAL : RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT + ); + if (randomBoolean()) { instance.setSourceRemoteStoreRepository(randomAlphaOfLengthBetween(5, 10)); } diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java index 0ff9d6f07751a..53f1d71947f7c 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java @@ -231,6 +231,49 @@ public void testShardsMatchingPredicateCount() { assertThat(clusterState.routingTable().shardsMatchingPredicateCount(predicate), is(2)); } + public void testAllShardsMatchingPredicate() { + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .build(); + ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build()) + .build(); + clusterState = ClusterState.builder(clusterState) + .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))) + .build(); + clusterState = allocation.reroute(clusterState, "reroute"); + + Predicate predicate = s -> s.state() == ShardRoutingState.UNASSIGNED && s.unassignedInfo().isDelayed(); + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(predicate).size(), is(0)); + + // starting primaries + clusterState = startInitializingShardsAndReroute(allocation, clusterState); + // starting replicas + clusterState = startInitializingShardsAndReroute(allocation, clusterState); + // remove node2 and reroute + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); + // make sure both replicas are marked as delayed (i.e. not reallocated) + clusterState = allocation.disassociateDeadNodes(clusterState, true, "reroute"); + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(predicate).size(), is(2)); + + // Verifies true against all shards on the node (active/inactive) + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(shard -> true).size(), is(4)); + // Verifies false against all shards on the node (active/inactive) + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(shard -> false).size(), is(0)); + // Verifies against all primary shards on the node + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(ShardRouting::primary).size(), is(2)); + // Verifies a predicate which tests for inactive replicas + assertThat( + clusterState.routingTable() + .allShardsSatisfyingPredicate(shardRouting -> !shardRouting.primary() && !shardRouting.active()) + .size(), + is(2) + ); + } + public void testActivePrimaryShardsGrouped() { assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0)); assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], false).size(), is(0)); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 2180a14f5bf30..bde8a45359814 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -69,6 +69,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.EmptySnapshotsInfoService; @@ -405,6 +406,7 @@ public void testFileCacheRemoteShardsDecisions() { DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings); Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(remoteIndexSettings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .persistentSettings(Settings.builder().put(FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.getKey(), 5).build()) .build(); RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index a121a190096b4..e4dec5163c400 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.mockito.Mockito; import org.opensearch.ExceptionsHelper; import org.opensearch.Version; import org.opensearch.action.ActionListener; @@ -105,6 +106,8 @@ import org.opensearch.client.AdminClient; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterInfo; +import org.opensearch.cluster.ClusterInfoService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -176,7 +179,9 @@ import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheCleaner; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; @@ -213,6 +218,7 @@ import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.RemoteTransportException; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportInterceptor; import org.opensearch.transport.TransportRequest; @@ -245,6 +251,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.mockito.Mockito.when; import static org.opensearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.opensearch.env.Environment.PATH_HOME_SETTING; import static org.opensearch.monitor.StatusInfo.Status.HEALTHY; @@ -260,6 +267,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; +import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING; public class SnapshotResiliencyTests extends OpenSearchTestCase { @@ -413,6 +421,106 @@ public void testSuccessfulSnapshotAndRestore() { assertEquals(0, snapshotInfo.failedShards()); } + public void testSearchableSnapshotOverSubscription() { + setupTestCluster(1, 2, 2); + + String repoName = "repo"; + String snapshotName = "snapshot"; + final String index = "test"; + final int shards = randomIntBetween(1, 10); + final int documents = randomIntBetween(0, 100); + + final TestClusterNodes.TestClusterNode clusterManagerNode = testClusterNodes.currentClusterManager( + testClusterNodes.nodes.values().iterator().next().clusterService.state() + ); + + Map nodeFileCacheStats = new HashMap<>(); + for (TestClusterNodes.TestClusterNode node : testClusterNodes.nodes.values()) { + nodeFileCacheStats.put(node.node.getId(), new FileCacheStats(0, 1, 0, 0, 0, 0, 0)); + } + ClusterInfo clusterInfo = new ClusterInfo(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), nodeFileCacheStats); + testClusterNodes.nodes.values().forEach(node -> when(node.getMockClusterInfoService().getClusterInfo()).thenReturn(clusterInfo)); + + final StepListener createSnapshotResponseListener = new StepListener<>(); + + continueOrDie(createRepoAndIndex(repoName, index, shards), createIndexResponse -> { + final Runnable afterIndexing = () -> client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .execute(createSnapshotResponseListener); + if (documents == 0) { + afterIndexing.run(); + } else { + final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + bulkRequest.add(new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i))); + } + final StepListener bulkResponseStepListener = new StepListener<>(); + client().bulk(bulkRequest, bulkResponseStepListener); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + afterIndexing.run(); + }); + } + }); + + final StepListener deleteIndexListener = new StepListener<>(); + + continueOrDie( + createSnapshotResponseListener, + createSnapshotResponse -> client().admin().indices().delete(new DeleteIndexRequest(index), deleteIndexListener) + ); + + final StepListener restoreSnapshotResponseListener = new StepListener<>(); + continueOrDie( + deleteIndexListener, + ignored -> client().admin() + .cluster() + .restoreSnapshot( + new RestoreSnapshotRequest(repoName, snapshotName).waitForCompletion(true) + .storageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT), + restoreSnapshotResponseListener + ) + ); + + final AtomicBoolean exceptionVerified = new AtomicBoolean(); + + restoreSnapshotResponseListener.whenComplete(null, restoreSnapshotException -> { + Throwable throwable = restoreSnapshotException; + if (restoreSnapshotException instanceof RemoteTransportException) { + throwable = restoreSnapshotException.getCause(); + } + try { + assertTrue(throwable instanceof SnapshotRestoreException); + assertTrue( + throwable.getMessage() + .contains( + "Size of the indexes to be restored exceeds the file cache bounds. Increase the file cache capacity on the cluster nodes using " + + NODE_SEARCH_CACHE_SIZE_SETTING.getKey() + + " setting." + ) + ); + } catch (SnapshotRestoreException ignored) {} + exceptionVerified.set(true); + }); + + runUntil(exceptionVerified::get, TimeUnit.MINUTES.toMillis(5L)); + assertTrue(exceptionVerified.get()); + SnapshotsInProgress finalSnapshotsInProgress = clusterManagerNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false)); + final Repository repository = clusterManagerNode.repositoriesService.repository(repoName); + Collection snapshotIds = getRepositoryData(repository).getSnapshotIds(); + assertThat(snapshotIds, hasSize(1)); + + final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotIds.iterator().next()); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); + assertThat(snapshotInfo.indices(), containsInAnyOrder(index)); + assertEquals(shards, snapshotInfo.successfulShards()); + assertEquals(0, snapshotInfo.failedShards()); + } + public void testSnapshotWithNodeDisconnects() { final int dataNodes = randomIntBetween(2, 10); final int clusterManagerNodes = randomFrom(1, 3, 5); @@ -1415,6 +1523,11 @@ private void setupTestCluster(int clusterManagerNodes, int dataNodes) { startCluster(); } + private void setupTestCluster(int clusterManagerNodes, int dataNodes, int searchNodes) { + testClusterNodes = new TestClusterNodes(clusterManagerNodes, dataNodes, searchNodes); + startCluster(); + } + private void scheduleSoon(Runnable runnable) { deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), runnable); } @@ -1465,6 +1578,7 @@ private Environment createEnvironment(String nodeName) { ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING.getKey(), ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING.get(Settings.EMPTY) ) + .put(FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.getKey(), 5) .put(MappingUpdatedAction.INDICES_MAX_IN_FLIGHT_UPDATES_SETTING.getKey(), 1000) // o.w. some tests might block .build() ); @@ -1488,6 +1602,10 @@ private final class TestClusterNodes { private final Set disconnectedNodes = new HashSet<>(); TestClusterNodes(int clusterManagerNodes, int dataNodes) { + this(clusterManagerNodes, dataNodes, 0); + } + + TestClusterNodes(int clusterManagerNodes, int dataNodes, int searchNodes) { for (int i = 0; i < clusterManagerNodes; ++i) { nodes.computeIfAbsent("node" + i, nodeName -> { try { @@ -1506,6 +1624,15 @@ private final class TestClusterNodes { } }); } + for (int i = 0; i < searchNodes; ++i) { + nodes.computeIfAbsent("search-node" + i, nodeName -> { + try { + return newSearchNode(nodeName); + } catch (IOException e) { + throw new AssertionError(e); + } + }); + } } public TestClusterNode nodeById(final String nodeId) { @@ -1524,6 +1651,10 @@ private TestClusterNode newDataNode(String nodeName) throws IOException { return newNode(nodeName, DiscoveryNodeRole.DATA_ROLE); } + private TestClusterNode newSearchNode(String nodeName) throws IOException { + return newNode(nodeName, DiscoveryNodeRole.SEARCH_ROLE); + } + private TestClusterNode newNode(String nodeName, DiscoveryNodeRole role) throws IOException { return new TestClusterNode( new DiscoveryNode( @@ -1667,6 +1798,8 @@ private final class TestClusterNode { private final ThreadPool threadPool; + private final ClusterInfoService clusterInfoService; + private Coordinator coordinator; TestClusterNode(DiscoveryNode node) throws IOException { @@ -1784,6 +1917,7 @@ public void onFailure(final Exception e) { final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); client = new NodeClient(settings, threadPool); + clusterInfoService = Mockito.mock(ClusterInfoService.class); final SetOnce rerouteServiceSetOnce = new SetOnce<>(); final SnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, @@ -1993,8 +2127,9 @@ public void onFailure(final Exception e) { new SystemIndices(emptyMap()), null ), - clusterSettings, - shardLimitValidator + shardLimitValidator, + indicesService, + clusterInfoService::getClusterInfo ); actions.put( PutMappingAction.INSTANCE, @@ -2207,6 +2342,10 @@ protected void assertSnapshotOrGenericThread() { } } + public ClusterInfoService getMockClusterInfoService() { + return clusterInfoService; + } + public void restart() { testClusterNodes.disconnectNode(this); final ClusterState oldState = this.clusterService.state(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index cdaa42c592c06..14275d838e6a9 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -117,6 +117,7 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.plugins.AnalysisPlugin; @@ -1201,6 +1202,7 @@ public static Settings.Builder settings(Version version) { public static Settings.Builder remoteIndexSettings(Version version) { Settings.Builder builder = Settings.builder() + .put(FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.getKey(), 5) .put(IndexMetadata.SETTING_VERSION_CREATED, version) .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.REMOTE_SNAPSHOT.getSettingsKey()); return builder;