Skip to content

Commit

Permalink
Merge branch 'main' into CoordinatorStats
Browse files Browse the repository at this point in the history
Signed-off-by: sahil <[email protected]>
  • Loading branch information
buddharajusahil authored Jul 31, 2023
2 parents cefc8bd + dea6889 commit 8d84ae3
Show file tree
Hide file tree
Showing 3,459 changed files with 26,918 additions and 13,803 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
2 changes: 2 additions & 0 deletions .ci/bwcVersions
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,5 @@ BWC_VERSION:
- "2.8.0"
- "2.8.1"
- "2.9.0"
- "2.9.1"
- "2.10.0"
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1 +1 @@
* @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale
* @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale @sohami
12 changes: 8 additions & 4 deletions .github/workflows/version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@ jobs:
CURRENT_VERSION_ARRAY[2]=$((CURRENT_VERSION_ARRAY[2]+1))
NEXT_VERSION=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}")
NEXT_VERSION_UNDERSCORE=$(IFS=_ ; echo "V_${CURRENT_VERSION_ARRAY[*]:0:3}")
NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99")
if [[ ${#CURRENT_VERSION_ARRAY[2]} -gt 1 ]]; then
NEXT_VERSION_ID="${CURRENT_VERSION_ARRAY[0]:0:3}0${CURRENT_VERSION_ARRAY[1]:0:3}${CURRENT_VERSION_ARRAY[2]:0:3}99"
else
NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99")
fi
echo "TAG=$TAG" >> $GITHUB_ENV
echo "BASE=$BASE" >> $GITHUB_ENV
echo "BASE_X=$BASE_X" >> $GITHUB_ENV
Expand All @@ -50,8 +54,8 @@ jobs:
echo " - \"$CURRENT_VERSION\"" >> .ci/bwcVersions
sed -i "s/opensearch = $CURRENT_VERSION/opensearch = $NEXT_VERSION/g" buildSrc/version.properties
echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" server/src/main/java/org/opensearch/Version.java
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
Expand All @@ -78,7 +82,7 @@ jobs:
echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
sed -i "s/- \"$CURRENT_VERSION\"/\0\n - \"$NEXT_VERSION\"/g" .ci/bwcVersions
echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
Expand Down
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ out/
benchmarks/src/main/generated/*
benchmarks/bin/*
benchmarks/build-eclipse-default/*
server/bin/*
server/build-eclipse-default/*
test/framework/build-eclipse-default/*

# eclipse files
.project
Expand Down Expand Up @@ -61,4 +64,4 @@ testfixtures_shared/
.ci/jobs/

# build files generated
doc-tools/missing-doclet/bin/
doc-tools/missing-doclet/bin/
115 changes: 34 additions & 81 deletions CHANGELOG.md

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion DEVELOPER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,10 @@ This repository is split into many top level directories. The most important one

### `distribution`

Builds our tar and zip archives and our rpm and deb packages.
Builds our tar and zip archives and our rpm and deb packages. There are several flavors of the distributions, with the classifier included in the name of the final deliverable (archive or package):
- default (no classifier), the distribution with bundled JDK
- `-no-jdk-` - the distribution without bundled JDK/JRE, assumes the JDK/JRE is going to be pre-installed on the target systems
- `-jre-` - the distribution bundled with JRE (smaller footprint), supported as experimental feature for some platforms

### `libs`

Expand Down
1 change: 1 addition & 0 deletions MAINTAINERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
| Sachin Kale | [sachinpkale](https://github.com/sachinpkale) | Amazon |
| Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon |
| Shweta Thareja | [shwetathareja](https://github.com/shwetathareja) | Amazon |
| Sorabh Hamirwasia | [sohami](https://github.com/sohami) | Amazon |
| Suraj Singh | [dreamer-89](https://github.com/dreamer-89) | Amazon |
| Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
| Vacha Shah | [VachaShah](https://github.com/VachaShah) | Amazon |
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
[![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
[![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
[![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
[![2.8 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.8.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.8.0")
[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0")
[![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
[![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
[![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
Expand Down
5 changes: 5 additions & 0 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
- [Test groups](#test-groups)
- [Load balancing and caches](#load-balancing-and-caches)
- [Test compatibility](#test-compatibility)
- [Retries](#retries)
- [Miscellaneous](#miscellaneous)
- [Running verification tasks](#running-verification-tasks)
- [Testing the REST layer](#testing-the-rest-layer)
Expand Down Expand Up @@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to

./gradlew test -Dtests.compatibility=1.0.0

## Retries

The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label.

## Miscellaneous

Run all tests without stopping on errors (inspect log files).
Expand Down
10 changes: 8 additions & 2 deletions benchmarks/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,16 @@ import org.opensearch.gradle.info.BuildParams

apply plugin: 'opensearch.build'
apply plugin: 'application'
mainClassName = 'org.openjdk.jmh.Main'

assemble.enabled = false
archivesBaseName = 'opensearch-benchmarks'

application {
mainClass = 'org.openjdk.jmh.Main'
}

base {
archivesBaseName = 'opensearch-benchmarks'
}

test.enabled = false

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@
import org.opensearch.action.search.SearchRequest;
import org.opensearch.common.breaker.CircuitBreaker;
import org.opensearch.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.Index;
import org.opensearch.index.shard.ShardId;
import org.opensearch.core.index.Index;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.SearchModule;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

import org.apache.lucene.util.BytesRef;
import org.opensearch.common.io.stream.DelayableWriteable;
import org.opensearch.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.aggregations.BucketOrder;
import org.opensearch.search.aggregations.InternalAggregation;
Expand Down
86 changes: 82 additions & 4 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ plugins {
id 'opensearch.docker-support'
id 'opensearch.global-build-info'
id "com.diffplug.spotless" version "6.19.0" apply false
id "org.gradle.test-retry" version "1.5.3" apply false
id "org.gradle.test-retry" version "1.5.4" apply false
id "test-report-aggregation"
id 'jacoco-report-aggregation'
}
Expand Down Expand Up @@ -327,7 +327,7 @@ allprojects {
javadoc.options.addStringOption('Xwerror', '-quiet')
}
javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"]
javadoc.options.addStringOption("-release", targetCompatibility.majorVersion)
javadoc.options.addStringOption("-release", java.targetCompatibility.majorVersion)
}

// support for reproducible builds
Expand Down Expand Up @@ -375,7 +375,7 @@ allprojects {
} else {
// Link to non-shadowed dependant projects
project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
String externalLinkName = upstreamProject.archivesBaseName
String externalLinkName = upstreamProject.base.archivesBaseName
String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version
String projectRelativePath = project.relativePath(upstreamProject.buildDir)
project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/"
Expand Down Expand Up @@ -424,6 +424,9 @@ gradle.projectsEvaluated {
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
task.jvmArgs += ["-Djava.security.manager=allow"]
}
if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) {
task.jvmArgs += ["--add-modules=jdk.incubator.vector"]
}
}
}

Expand Down Expand Up @@ -452,7 +455,7 @@ gradle.projectsEvaluated {
testReportAggregation it
}
subprojects.findAll { it.pluginManager.hasPlugin('jacoco') }.forEach {
jacocoAggregation it
jacocoAggregation it
}
}
}
Expand All @@ -467,6 +470,81 @@ subprojects {
maxFailures = 10
}
failOnPassedAfterRetry = false
filter {
includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests")
includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT")
includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT")
includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT")
includeClasses.add("org.opensearch.aliases.IndexAliasesIT")
includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT")
includeClasses.add("org.opensearch.blocks.SimpleBlocksIT")
includeClasses.add("org.opensearch.client.PitIT")
includeClasses.add("org.opensearch.client.ReindexIT")
includeClasses.add("org.opensearch.cluster.ClusterHealthIT")
includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT")
includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT")
includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT")
includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests")
includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests")
includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT")
includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests")
includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT")
includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT")
includeClasses.add("org.opensearch.http.SearchRestCancellationIT")
includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests")
includeClasses.add("org.opensearch.index.IndexServiceTests")
includeClasses.add("org.opensearch.index.IndexSettingsTests")
includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT")
includeClasses.add("org.opensearch.index.ShardIndexingPressureIT")
includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT")
includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests")
includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests")
includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests")
includeClasses.add("org.opensearch.index.shard.IndexShardIT")
includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests")
includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests")
includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT")
includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests")
includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT")
includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT")
includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT")
includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests")
includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT")
includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests")
includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests")
includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep")
includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT")
includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT")
includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT")
includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT")
includeClasses.add("org.opensearch.remotestore.RemoteStoreIT")
includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT")
includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT")
includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT")
includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT")
includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT")
includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests")
includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests")
includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests")
includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests")
includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests")
includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT")
includeClasses.add("org.opensearch.search.SearchTimeoutIT")
includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT")
includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT")
includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT")
includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT")
includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT")
includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT")
includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT")
includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT")
includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT")
includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT")
includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT")
includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT")
includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
}
}
}
}
Expand Down
16 changes: 9 additions & 7 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -102,23 +102,23 @@ dependencies {

api localGroovy()

api 'commons-codec:commons-codec:1.15'
api 'commons-codec:commons-codec:1.16.0'
api 'org.apache.commons:commons-compress:1.23.0'
api 'org.apache.ant:ant:1.10.13'
api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
api 'com.netflix.nebula:gradle-info-plugin:12.1.4'
api 'com.netflix.nebula:gradle-info-plugin:12.1.6'
api 'org.apache.rat:apache-rat:0.15'
api 'commons-io:commons-io:2.13.0'
api "net.java.dev.jna:jna:5.13.0"
api 'gradle.plugin.com.github.johnrengelman:shadow:8.0.0'
api 'com.github.johnrengelman:shadow:8.1.1'
api 'org.jdom:jdom2:2.0.6.1'
api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
api 'de.thetaphi:forbiddenapis:3.5.1'
api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
api 'org.apache.maven:maven-model:3.9.2'
api 'com.networknt:json-schema-validator:1.0.85'
api 'org.apache.maven:maven-model:3.9.3'
api 'com.networknt:json-schema-validator:1.0.86'
api 'org.jruby.jcodings:jcodings:1.0.58'
api 'org.jruby.joni:joni:2.2.1'
api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
Expand Down Expand Up @@ -164,8 +164,10 @@ if (project != rootProject) {
apply plugin: 'opensearch.publish'

allprojects {
targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_11
java {
targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_11
}
}

// groovydoc succeeds, but has some weird internal exception...
Expand Down
Loading

0 comments on commit 8d84ae3

Please sign in to comment.