From f59e787ddfece83c3226df335946d65c4f24ab18 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Sat, 16 Sep 2023 12:46:22 -0500 Subject: [PATCH] feat(search): update to support OpenSearch 2.x --- build.gradle | 21 +- datahub-graphql-core/build.gradle | 1 + .../analytics/service/AnalyticsService.java | 38 +- .../graphql/resolvers/ResolverUtilsTest.java | 4 +- .../auth/ListAccessTokensResolverTest.java | 10 +- .../GetIngestionSourceResolverTest.java | 3 +- .../resolvers/mutate/SiblingsUtilsTest.java | 4 +- .../datahub/graphql/utils/DateUtilTest.java | 5 +- .../upgrade/config/NoCodeCleanupConfig.java | 2 +- .../DeleteLegacySearchIndicesStep.java | 6 +- .../nocodecleanup/NoCodeCleanupUpgrade.java | 2 +- .../steps/BuildIndicesPostStep.java | 4 +- .../steps/BuildIndicesPreStep.java | 10 +- .../elasticsearch/steps/CleanIndicesStep.java | 2 +- .../system/elasticsearch/util/IndexUtils.java | 12 +- docker/build.gradle | 10 + docker/docker-compose-with-cassandra.yml | 7 +- docker/docker-compose-without-neo4j.yml | 7 +- docker/docker-compose.yml | 7 +- docker/elasticsearch/env/docker.env | 1 + .../docker-compose-m1.quickstart.yml | 7 +- ...er-compose-without-neo4j-m1.quickstart.yml | 7 +- ...ocker-compose-without-neo4j.quickstart.yml | 7 +- .../quickstart/docker-compose.quickstart.yml | 7 +- li-utils/build.gradle | 1 + .../mxe-utils-avro-1.7/build.gradle | 1 + .../java/datahub-client/build.gradle | 1 + .../resources/MetadataChangeProposal.avsc | 5 + .../datahub/protobuf/ProtobufDatasetTest.java | 2 +- .../datahub/protobuf/ProtobufUtilsTest.java | 2 +- .../protobuf/model/ProtobufEnumTest.java | 2 +- .../protobuf/model/ProtobufFieldTest.java | 2 +- .../protobuf/model/ProtobufGraphTest.java | 2 +- .../protobuf/model/ProtobufMessageTest.java | 2 +- .../model/ProtobufOneOfFieldTest.java | 2 +- .../protobuf/visitors/VisitContextTest.java | 2 +- .../visitors/dataset/DatasetVisitorTest.java | 2 +- .../dataset/DescriptionVisitorTest.java | 2 +- .../visitors/dataset/DomainVisitorTest.java | 2 +- .../InstitutionalMemoryVisitorTest.java | 2 +- .../KafkaTopicPropertyVisitorTest.java | 2 +- .../dataset/OwnershipVisitorTest.java | 2 +- .../visitors/dataset/PropertyVisitorTest.java | 2 +- .../dataset/TermAssociationVisitorTest.java | 2 +- .../ProtobufExtensionFieldVisitorTest.java | 2 +- .../field/SchemaFieldVisitorTest.java | 2 +- .../protobuf/visitors/tag/TagVisitorTest.java | 2 +- metadata-io/build.gradle | 15 +- .../graph/elastic/ESGraphQueryDAO.java | 38 +- .../graph/elastic/ESGraphWriteDAO.java | 10 +- .../elastic/ElasticSearchGraphService.java | 4 +- .../graph/elastic/TimeFilterUtils.java | 6 +- .../candidatesource/MostPopularSource.java | 22 +- .../candidatesource/RecentlyEditedSource.java | 24 +- .../candidatesource/RecentlyViewedSource.java | 24 +- .../elasticsearch/ElasticSearchService.java | 2 +- .../indexbuilder/ESIndexBuilder.java | 74 ++-- .../indexbuilder/ReindexConfig.java | 2 +- .../elasticsearch/query/ESBrowseDAO.java | 30 +- .../elasticsearch/query/ESSearchDAO.java | 32 +- .../request/AggregationQueryBuilder.java | 4 +- .../request/AutocompleteRequestHandler.java | 18 +- .../query/request/PITAwareSearchRequest.java | 4 +- .../query/request/SearchQueryBuilder.java | 42 +- .../query/request/SearchRequestHandler.java | 34 +- .../elasticsearch/update/BulkListener.java | 12 +- .../elasticsearch/update/ESBulkProcessor.java | 26 +- .../elasticsearch/update/ESWriteDAO.java | 18 +- .../metadata/search/utils/ESUtils.java | 28 +- .../systemmetadata/ESSystemMetadataDAO.java | 40 +- .../ElasticSearchSystemMetadataService.java | 16 +- .../ElasticSearchTimeseriesAspectService.java | 40 +- .../TimeseriesAspectIndexBuilders.java | 2 +- .../elastic/query/ESAggregatedStatsDAO.java | 36 +- .../metadata/ESTestConfiguration.java | 153 -------- .../com/linkedin/metadata/EbeanTestUtils.java | 19 +- .../update/BulkListenerTest.java | 4 +- .../update/ESBulkProcessorTest.java | 2 +- .../entity/EbeanEntityServiceTest.java | 11 +- .../ESGraphQueryDAOTest.java | 5 +- .../SearchGraphServiceTestBase.java} | 44 ++- .../TimeFilterUtilsTest.java | 6 +- .../SearchGraphServiceElasticSearchTest.java | 49 +++ .../SearchGraphServiceOpenSearchTest.java | 48 +++ ...eTest.java => LineageServiceTestBase.java} | 112 +++--- ...ceTest.java => SearchServiceTestBase.java} | 70 ++-- ...rviceTest.java => TestEntityTestBase.java} | 66 ++-- .../elasticsearch/ElasticSearchSuite.java | 32 ++ .../GoldenElasticSearchTest.java | 44 +++ .../IndexBuilderElasticSearchTest.java | 30 ++ .../LineageDataFixtureElasticSearchTest.java | 43 ++ .../LineageServiceElasticSearchTest.java | 66 ++++ .../SampleDataFixtureElasticSearchTest.java | 45 +++ .../SearchDAOElasticSearchTest.java | 35 ++ .../SearchServiceElasticSearchTest.java | 65 ++++ ...ystemMetadataServiceElasticSearchTest.java | 47 +++ .../TestEntityElasticSearchTest.java | 65 ++++ ...eseriesAspectServiceElasticSearchTest.java | 46 +++ .../elasticsearch/query/ESSearchDAOTest.java | 312 --------------- .../GoldenTestBase.java} | 65 ++-- .../LineageDataFixtureTestBase.java} | 39 +- .../SampleDataFixtureTestBase.java} | 366 +++++++++--------- .../IndexBuilderTestBase.java} | 65 ++-- .../indexbuilder/MappingsBuilderTest.java | 4 +- .../opensearch/GoldenOpenSearchTest.java | 44 +++ .../IndexBuilderOpenSearchTest.java | 30 ++ .../LineageDataFixtureOpenSearchTest.java | 43 ++ .../LineageServiceOpenSearchTest.java | 65 ++++ .../search/opensearch/OpenSearchSuite.java | 31 ++ .../SampleDataFixtureOpenSearchTest.java | 44 +++ .../opensearch/SearchDAOOpenSearchTest.java | 33 ++ .../SearchServiceOpenSearchTest.java | 65 ++++ .../SystemMetadataServiceOpenSearchTest.java | 47 +++ .../opensearch/TestEntityOpenSearchTest.java | 65 ++++ ...TimeseriesAspectServiceOpenSearchTest.java | 46 +++ .../BrowseDAOTest.java} | 19 +- .../search/query/SearchDAOTestBase.java | 307 +++++++++++++++ .../request/AggregationQueryBuilderTest.java | 6 +- .../AutocompleteRequestHandlerTest.java | 18 +- .../request/CustomizedQueryHandlerTest.java | 16 +- .../query/request/SearchQueryBuilderTest.java | 26 +- .../request/SearchRequestHandlerTest.java | 29 +- .../metadata/search/utils/ESUtilsTest.java | 2 +- ...ava => SystemMetadataServiceTestBase.java} | 41 +- .../TimeseriesAspectServiceTestBase.java} | 43 +- .../test/fixtures/elasticsearch/Utils.java | 22 -- .../test/DataGenerator.java | 5 +- .../test/fixtures/search}/EntityExporter.java | 18 +- .../test/fixtures/search}/FixtureReader.java | 10 +- .../test/fixtures/search}/FixtureWriter.java | 24 +- .../fixtures/search}/LineageExporter.java | 14 +- .../SampleDataFixtureConfiguration.java} | 21 +- .../fixtures/search/SearchFixtureUtils.java} | 51 ++- .../SearchLineageFixtureConfiguration.java} | 16 +- .../test/models/Anonymized.java | 2 +- .../test/models/DatasetAnonymized.java | 2 +- .../test/models/GraphAnonymized.java | 4 +- .../search/ElasticsearchTestContainer.java | 42 ++ .../test/search/OpenSearchTestContainer.java | 43 ++ .../test/search/SearchTestContainer.java | 14 + .../test/search/SearchTestUtils.java} | 60 ++- .../config/SearchCommonTestConfiguration.java | 63 +++ .../SearchTestContainerConfiguration.java | 88 +++++ .../src/test/resources/testng-other.xml | 14 + .../src/test/resources/testng-search.xml | 16 + metadata-io/src/test/resources/testng.xml | 14 + metadata-jobs/mae-consumer/build.gradle | 1 + .../kafka/elasticsearch/ElasticEvent.java | 2 +- .../elasticsearch/ElasticsearchConnector.java | 10 +- .../kafka/elasticsearch/JsonElasticEvent.java | 12 +- .../kafka/elasticsearch/MCEElasticEvent.java | 12 +- metadata-models/build.gradle | 1 + metadata-service/auth-impl/build.gradle | 1 + .../common/RestHighLevelClientFactory.java | 6 +- .../factory/graphql/GraphQLEngineFactory.java | 2 +- .../MostPopularCandidateSourceFactory.java | 2 +- .../RecentlyEditedCandidateSourceFactory.java | 2 +- ...ecentlySearchedCandidateSourceFactory.java | 2 +- .../RecentlyViewedCandidateSourceFactory.java | 2 +- .../BaseElasticSearchComponentsFactory.java | 2 +- .../ElasticSearchBulkProcessorFactory.java | 4 +- .../ElasticSearchIndexBuilderFactory.java | 2 +- .../gms/factory/telemetry/DailyReport.java | 2 +- .../telemetry/ScheduledAnalyticsFactory.java | 2 +- ...ElasticSearchBulkProcessorFactoryTest.java | 2 +- .../telemetry/TelemetryUtilsTest.java | 2 +- .../controller/HealthCheckController.java | 10 +- .../OpenAPIAnalyticsTestConfiguration.java | 2 +- .../openapi/util/OpenApiEntitiesUtilTest.java | 2 +- .../elastic/OperationsController.java | 2 +- .../operations/OperationsResource.java | 2 +- .../operations/OperationsResourceTest.java | 5 +- metadata-service/services/build.gradle | 2 - .../RecentlySearchedSource.java | 24 +- .../systemmetadata/SystemMetadataService.java | 2 +- .../gms/servlet/ConfigSearchExport.java | 20 +- .../java/com/datahub/gms/util/CSVWriter.java | 4 +- metadata-utils/build.gradle | 1 + .../linkedin/metadata/utils/SearchUtil.java | 4 +- smoke-test/cypress-dev.sh | 3 +- smoke-test/run-quickstart.sh | 9 + smoke-test/set-cypress-creds.sh | 3 +- .../tests/cypress/cypress/e2e/login/login.js | 2 +- 183 files changed, 2933 insertions(+), 1589 deletions(-) delete mode 100644 metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java rename metadata-io/src/test/java/com/linkedin/metadata/graph/{elastic => search}/ESGraphQueryDAOTest.java (98%) rename metadata-io/src/test/java/com/linkedin/metadata/graph/{elastic/ElasticSearchGraphServiceTest.java => search/SearchGraphServiceTestBase.java} (93%) rename metadata-io/src/test/java/com/linkedin/metadata/graph/{elastic => search}/TimeFilterUtilsTest.java (82%) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java rename metadata-io/src/test/java/com/linkedin/metadata/search/{LineageSearchServiceTest.java => LineageServiceTestBase.java} (94%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{SearchServiceTest.java => SearchServiceTestBase.java} (92%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch/ElasticSearchServiceTest.java => TestEntityTestBase.java} (86%) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchSuite.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java delete mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch/fixtures/ElasticSearchGoldenTest.java => fixtures/GoldenTestBase.java} (74%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch/fixtures/SearchLineageDataFixtureTests.java => fixtures/LineageDataFixtureTestBase.java} (52%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch/fixtures/SampleDataFixtureTests.java => fixtures/SampleDataFixtureTestBase.java} (81%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch/indexbuilder/ESIndexBuilderTest.java => indexbuilder/IndexBuilderTestBase.java} (85%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch => }/indexbuilder/MappingsBuilderTest.java (98%) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/OpenSearchSuite.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch/query/ESBrowseDAOTest.java => query/BrowseDAOTest.java} (86%) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch => }/query/request/AggregationQueryBuilderTest.java (94%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch => }/query/request/AutocompleteRequestHandlerTest.java (88%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch => }/query/request/CustomizedQueryHandlerTest.java (93%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch => }/query/request/SearchQueryBuilderTest.java (95%) rename metadata-io/src/test/java/com/linkedin/metadata/search/{elasticsearch => }/query/request/SearchRequestHandlerTest.java (95%) rename metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/{ElasticSearchSystemMetadataServiceTest.java => SystemMetadataServiceTestBase.java} (84%) rename metadata-io/src/test/java/com/linkedin/metadata/timeseries/{elastic/ElasticSearchTimeseriesAspectServiceTest.java => search/TimeseriesAspectServiceTestBase.java} (97%) delete mode 100644 metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/Utils.java rename metadata-io/src/test/java/io/{datahub => datahubproject}/test/DataGenerator.java (99%) rename metadata-io/src/test/java/io/{datahub/test/fixtures/elasticsearch => datahubproject/test/fixtures/search}/EntityExporter.java (81%) rename metadata-io/src/test/java/io/{datahub/test/fixtures/elasticsearch => datahubproject/test/fixtures/search}/FixtureReader.java (93%) rename metadata-io/src/test/java/io/{datahub/test/fixtures/elasticsearch => datahubproject/test/fixtures/search}/FixtureWriter.java (75%) rename metadata-io/src/test/java/io/{datahub/test/fixtures/elasticsearch => datahubproject/test/fixtures/search}/LineageExporter.java (95%) rename metadata-io/src/test/java/{com/linkedin/metadata/ESSampleDataFixture.java => io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java} (94%) rename metadata-io/src/test/java/{com/linkedin/metadata/ESTestFixtureUtils.java => io/datahubproject/test/fixtures/search/SearchFixtureUtils.java} (67%) rename metadata-io/src/test/java/{com/linkedin/metadata/ESSearchLineageFixture.java => io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java} (95%) rename metadata-io/src/test/java/io/{datahub => datahubproject}/test/models/Anonymized.java (97%) rename metadata-io/src/test/java/io/{datahub => datahubproject}/test/models/DatasetAnonymized.java (97%) rename metadata-io/src/test/java/io/{datahub => datahubproject}/test/models/GraphAnonymized.java (82%) create mode 100644 metadata-io/src/test/java/io/datahubproject/test/search/ElasticsearchTestContainer.java create mode 100644 metadata-io/src/test/java/io/datahubproject/test/search/OpenSearchTestContainer.java create mode 100644 metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java rename metadata-io/src/test/java/{com/linkedin/metadata/ESTestUtils.java => io/datahubproject/test/search/SearchTestUtils.java} (74%) create mode 100644 metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java create mode 100644 metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java create mode 100644 metadata-io/src/test/resources/testng-other.xml create mode 100644 metadata-io/src/test/resources/testng-search.xml create mode 100644 metadata-io/src/test/resources/testng.xml diff --git a/build.gradle b/build.gradle index 1b6b82d51c2d4a..07a0e6ad1f49fd 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ buildscript { ext.openTelemetryVersion = '1.18.0' ext.neo4jVersion = '4.4.9' ext.testContainersVersion = '1.17.4' - ext.elasticsearchVersion = '7.10.2' + ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x ext.jacksonVersion = '2.15.2' ext.jettyVersion = '9.4.46.v20220331' ext.playVersion = '2.8.18' @@ -90,15 +90,15 @@ project.ext.externalDependency = [ 'ebean': 'io.ebean:ebean:' + ebeanVersion, 'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion, 'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion, - 'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:' + elasticsearchVersion, - 'elasticSearchTransport': 'org.elasticsearch.client:transport:' + elasticsearchVersion, + 'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion, + 'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0', 'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1', 'graphqlJava': 'com.graphql-java:graphql-java:19.5', 'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1', 'gson': 'com.google.code.gson:gson:2.8.9', 'guice': 'com.google.inject:guice:4.2.3', 'guava': 'com.google.guava:guava:32.1.2-jre', - 'h2': 'com.h2database:h2:2.1.214', + 'h2': 'com.h2database:h2:2.2.224', 'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2', 'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2', "hadoopClient": "org.apache.hadoop:hadoop-client:$hadoop3Version", @@ -202,13 +202,15 @@ project.ext.externalDependency = [ 'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion", 'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12', 'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41', - 'testng': 'org.testng:testng:7.3.0', + 'testngJava8': 'org.testng:testng:7.5.1', + 'testng': 'org.testng:testng:7.8.0', 'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion, 'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion, 'testContainersPostgresql':'org.testcontainers:postgresql:' + testContainersVersion, 'testContainersElasticsearch': 'org.testcontainers:elasticsearch:' + testContainersVersion, 'testContainersCassandra': 'org.testcontainers:cassandra:' + testContainersVersion, 'testContainersKafka': 'org.testcontainers:kafka:' + testContainersVersion, + 'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0', 'typesafeConfig':'com.typesafe:config:1.4.1', 'wiremock':'com.github.tomakehurst:wiremock:2.10.0', 'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14', @@ -257,7 +259,6 @@ subprojects { plugins.withType(JavaPlugin) { dependencies { - testImplementation externalDependency.testng constraints { implementation('io.netty:netty-all:4.1.86.Final') implementation('org.apache.commons:commons-compress:1.21') @@ -268,12 +269,6 @@ subprojects { } } - tasks.withType(Test) { - if (!name.startsWith('integ')) { - useTestNG() - } - } - checkstyle { configDirectory = file("${project.rootDir}/gradle/checkstyle") sourceSets = [ getProject().sourceSets.main, getProject().sourceSets.test ] @@ -292,6 +287,8 @@ subprojects { javaLauncher = javaToolchains.launcherFor { languageVersion = JavaLanguageVersion.of(11) } + // https://docs.gradle.org/current/userguide/performance.html + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 } afterEvaluate { diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 89ba8f17b6aebd..fba0031351b588 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -24,6 +24,7 @@ dependencies { annotationProcessor externalDependency.lombok testImplementation externalDependency.mockito + testImplementation externalDependency.testng } graphqlCodegen { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java index 44b1779f8b006b..4135a7b0da1482 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java @@ -20,25 +20,25 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.Cardinality; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.opensearch.search.aggregations.bucket.filter.Filter; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.Cardinality; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java index c391615db92688..7cd548a4790bae 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java @@ -15,7 +15,6 @@ import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import graphql.schema.DataFetchingEnvironment; -import junit.framework.TestCase; import org.testng.annotations.Test; import org.mockito.Mockito; @@ -24,9 +23,10 @@ import java.util.stream.Collectors; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static org.testng.AssertJUnit.assertEquals; -public class ResolverUtilsTest extends TestCase { +public class ResolverUtilsTest { @Test public void testCriterionFromFilter() throws Exception { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java index 8c23335b7e9d3c..54b8d23bab301d 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java @@ -10,15 +10,15 @@ import com.linkedin.metadata.Constants; import graphql.schema.DataFetchingEnvironment; import java.util.Collections; -import junit.framework.TestCase; import org.mockito.Mockito; +import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; -public class ListAccessTokensResolverTest extends TestCase { +public class ListAccessTokensResolverTest { -// @Test + @Test public void testGetSuccess() throws Exception { final DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final QueryContext mockAllowContext = TestUtils.getMockAllowContext(); @@ -36,13 +36,13 @@ public void testGetSuccess() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); final EntityClient mockClient = Mockito.mock(EntityClient.class); - Mockito.when(mockClient.filter( + Mockito.when(Mockito.eq(mockClient.filter( Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME), Mockito.eq(buildFilter(filters, Collections.emptyList())), Mockito.notNull(), Mockito.eq(input.getStart()), Mockito.eq(input.getCount()), - Mockito.eq(getAuthentication(mockEnv)))) + Mockito.eq(getAuthentication(mockEnv))))) .thenReturn(null); final ListAccessTokensResolver resolver = new ListAccessTokensResolver(mockClient); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java index 2d9f43029c479f..ebafd1782e000d 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java @@ -14,11 +14,12 @@ import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import java.util.HashSet; + import org.mockito.Mockito; import org.testng.annotations.Test; -import static org.testng.Assert.*; import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; +import static org.testng.Assert.assertThrows; public class GetIngestionSourceResolverTest { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java index d8325e9a747405..1adf7b1200574e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java @@ -6,7 +6,6 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.SiblingsUtils; import com.linkedin.metadata.entity.EntityService; -import junit.framework.TestCase; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -14,8 +13,9 @@ import java.util.Optional; import static com.linkedin.metadata.Constants.SIBLINGS_ASPECT_NAME; +import static org.testng.AssertJUnit.assertEquals; -public class SiblingsUtilsTest extends TestCase { +public class SiblingsUtilsTest { private static final String TEST_DATASET_URN1 = "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)"; private static final String TEST_DATASET_URN2 = "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created2,PROD)"; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java index 989ebc18e9f6c4..0a58ff88586c6d 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java @@ -1,12 +1,13 @@ package com.linkedin.datahub.graphql.utils; import com.linkedin.datahub.graphql.util.DateUtil; -import junit.framework.TestCase; import org.joda.time.DateTime; import org.mockito.Mockito; import org.testng.annotations.Test; -public class DateUtilTest extends TestCase { +import static org.testng.AssertJUnit.assertEquals; + +public class DateUtilTest { private DateTime setTimeParts(int dayOfMonth, boolean zeroTime) { DateTime result = new DateTime() diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java index 0fb8b0eb6e20f2..23ea81009fa1da 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java @@ -5,7 +5,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import io.ebean.Database; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java index 15bbe40d1e566a..9a64d5fe1810c9 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java @@ -7,9 +7,9 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import java.util.function.Function; import lombok.RequiredArgsConstructor; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; // Do we need SQL-tech specific migration paths? diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java index 2b5e23c5f82690..a5d8d6ce9b666f 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; public class NoCodeCleanupUpgrade implements Upgrade { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java index 465a5fe3426676..2feca1f27e6258 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java @@ -16,8 +16,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.client.RequestOptions; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.client.RequestOptions; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java index 6f2f3a8bd727c7..82b9428c89fb8f 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java @@ -19,10 +19,10 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.indices.ResizeRequest; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.indices.ResizeRequest; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; @@ -97,7 +97,7 @@ private boolean blockWrites(String indexName) throws InterruptedException, IOExc ack = _esComponents.getSearchClient().indices() .putSettings(request, RequestOptions.DEFAULT).isAcknowledged(); log.info("Updated index {} with new settings. Settings: {}, Acknowledged: {}", indexName, indexSettings, ack); - } catch (ElasticsearchStatusException | IOException ese) { + } catch (OpenSearchStatusException | IOException ese) { // Cover first run case, indices won't exist so settings updates won't work nor will the rest of the preConfigure steps. // Since no data are in there they are skippable. // Have to hack around HighLevelClient not sending the actual Java type nor having an easy way to extract it :( diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java index f60aa283c01405..bb042bac6df955 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java @@ -9,7 +9,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import java.util.List; import java.util.function.Function; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java index fa414798ccfea5..4b04feac62cbf3 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java @@ -4,12 +4,12 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.client.GetAliasesResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; import java.io.IOException; import java.util.ArrayList; diff --git a/docker/build.gradle b/docker/build.gradle index ae101fe1defc5f..0faea626e982d9 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -38,6 +38,16 @@ task quickstart(type: Exec, dependsOn: ':metadata-ingestion:install') { // environment "ACTIONS_VERSION", 'alpine3.17-slim' // environment "DATAHUB_ACTIONS_IMAGE", 'nginx' + // Elastic + // environment "DATAHUB_SEARCH_IMAGE", 'elasticsearch' + // environment "DATAHUB_SEARCH_TAG", '7.10.1' + + // OpenSearch + environment "DATAHUB_SEARCH_IMAGE", 'opensearchproject/opensearch' + environment "DATAHUB_SEARCH_TAG", '2.9.0' + environment "XPACK_SECURITY_ENABLED", 'plugins.security.disabled=true' + environment "USE_AWS_ELASTICSEARCH", 'true' + def cmd = [ 'source ../metadata-ingestion/venv/bin/activate && ', 'datahub docker quickstart', diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 08f8cc1ec9c45e..9543e67da07f2a 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -96,6 +96,9 @@ services: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile env_file: elasticsearch-setup/env/docker.env + environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} depends_on: elasticsearch: condition: service_healthy @@ -117,13 +120,13 @@ services: elasticsearch: container_name: elasticsearch hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - 9200:9200 env_file: elasticsearch/env/docker.env environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} healthcheck: test: curl -sS --fail http://elasticsearch:9200/_cluster/health?wait_for_status=yellow&timeout=0s start_period: 5s diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 0b2e4f76b8fa9e..022362782f7420 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -81,6 +81,9 @@ services: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile env_file: elasticsearch-setup/env/docker.env + environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} depends_on: elasticsearch: condition: service_healthy @@ -104,13 +107,13 @@ services: elasticsearch: container_name: elasticsearch hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 env_file: elasticsearch/env/docker.env environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} deploy: resources: limits: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d07ea5fa88f8b8..a486689e050a21 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -83,6 +83,9 @@ services: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile env_file: elasticsearch-setup/env/docker.env + environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} depends_on: elasticsearch: condition: service_healthy @@ -109,13 +112,13 @@ services: elasticsearch: container_name: elasticsearch hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 env_file: elasticsearch/env/docker.env environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} deploy: resources: limits: diff --git a/docker/elasticsearch/env/docker.env b/docker/elasticsearch/env/docker.env index 4b1f0215ea6c8d..46b5836dedd284 100644 --- a/docker/elasticsearch/env/docker.env +++ b/docker/elasticsearch/env/docker.env @@ -1 +1,2 @@ ES_JAVA_OPTS="-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" +OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" \ No newline at end of file diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 38418bc8c41b98..89e9aaa0defd61 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -161,8 +161,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -170,7 +171,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -181,6 +182,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index cf879faa6a3f0f..f6284edc83648e 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -154,8 +154,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -163,7 +164,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -174,6 +175,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 007830078d2b4a..4e3503e35c0db1 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -154,8 +154,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -163,7 +164,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -174,6 +175,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 390543b92123f1..e2f52064389e06 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -161,8 +161,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -170,7 +171,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -181,6 +182,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/li-utils/build.gradle b/li-utils/build.gradle index 8f526cffba094b..1d5222e39185af 100644 --- a/li-utils/build.gradle +++ b/li-utils/build.gradle @@ -28,6 +28,7 @@ dependencies { testImplementation externalDependency.commonsIo testImplementation project(':test-models') testImplementation project(path: ':test-models', configuration: 'testDataTemplate') + testImplementation externalDependency.testngJava8 } idea { diff --git a/metadata-events/mxe-utils-avro-1.7/build.gradle b/metadata-events/mxe-utils-avro-1.7/build.gradle index 82249d393578cb..3b137965d6c19f 100644 --- a/metadata-events/mxe-utils-avro-1.7/build.gradle +++ b/metadata-events/mxe-utils-avro-1.7/build.gradle @@ -5,6 +5,7 @@ dependencies { api project(':metadata-models') api spec.product.pegasus.dataAvro1_6 + testImplementation externalDependency.testng testImplementation project(':test-models') testImplementation project(path: ':test-models', configuration: 'testDataTemplate') diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index fc72fc4257491c..95de3cdb3c5262 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -49,6 +49,7 @@ dependencies { annotationProcessor externalDependency.lombok // VisibleForTesting compileOnly externalDependency.guava + testImplementation externalDependency.testngJava8 testImplementation externalDependency.mockito testImplementation externalDependency.mockServer testImplementation externalDependency.mockServerClient diff --git a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc index 6a723090fda07c..64216636af26d1 100644 --- a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc +++ b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc @@ -143,6 +143,11 @@ "type" : [ "string", "null" ], "doc" : "The last run id that produced the metadata. Populated in case of batch-ingestion.", "default" : "no-run-id-provided" + }, { + "name" : "pipelineName", + "type" : [ "null", "string" ], + "doc" : "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion.", + "default" : null }, { "name" : "registryName", "type" : [ "null", "string" ], diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufDatasetTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufDatasetTest.java index 748990752f45b9..bbb8e532f1033d 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufDatasetTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufDatasetTest.java @@ -26,7 +26,7 @@ import datahub.protobuf.model.ProtobufField; import datahub.protobuf.visitors.ProtobufModelVisitor; import datahub.protobuf.visitors.VisitContext; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Set; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufUtilsTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufUtilsTest.java index 58e78435a43a51..3a00edca8284a8 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufUtilsTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufUtilsTest.java @@ -3,7 +3,7 @@ import com.google.protobuf.DescriptorProtos; import com.google.protobuf.ExtensionRegistry; import datahub.protobuf.model.ProtobufGraph; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufEnumTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufEnumTest.java index 3696f5795e1f95..7c98077690d66f 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufEnumTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufEnumTest.java @@ -5,7 +5,7 @@ import com.google.protobuf.DescriptorProtos.FileDescriptorProto; import com.linkedin.schema.EnumType; import com.linkedin.schema.SchemaFieldDataType; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; import java.util.Set; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufFieldTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufFieldTest.java index a21acf7f6c1134..543b815f7f72b6 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufFieldTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufFieldTest.java @@ -17,7 +17,7 @@ import com.linkedin.schema.SchemaMetadata; import com.linkedin.schema.StringType; import datahub.protobuf.ProtobufDataset; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufGraphTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufGraphTest.java index a7e6dd035160c0..80ffafff3f4511 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufGraphTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufGraphTest.java @@ -1,7 +1,7 @@ package datahub.protobuf.model; import com.google.protobuf.DescriptorProtos.FileDescriptorSet; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.HashSet; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufMessageTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufMessageTest.java index 035c16552aeb5a..e961b6ffd2d619 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufMessageTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufMessageTest.java @@ -5,7 +5,7 @@ import com.linkedin.schema.MapType; import com.linkedin.schema.RecordType; import com.linkedin.schema.SchemaFieldDataType; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; import java.util.Set; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufOneOfFieldTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufOneOfFieldTest.java index f9b168437643bc..438e0a79206bd9 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufOneOfFieldTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufOneOfFieldTest.java @@ -6,7 +6,7 @@ import com.google.protobuf.DescriptorProtos.OneofDescriptorProto; import com.linkedin.schema.SchemaFieldDataType; import com.linkedin.schema.UnionType; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; import java.util.Set; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/VisitContextTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/VisitContextTest.java index 9645c6b66ef5f5..ceebefb3a207e1 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/VisitContextTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/VisitContextTest.java @@ -5,7 +5,7 @@ import datahub.protobuf.model.ProtobufElement; import datahub.protobuf.model.ProtobufGraph; import org.jgrapht.GraphPath; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java index 165823d8e4925f..fb51f42a6c7592 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java @@ -2,7 +2,7 @@ import com.linkedin.common.urn.DatasetUrn; import com.linkedin.data.template.RecordTemplate; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URISyntaxException; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DescriptionVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DescriptionVisitorTest.java index c5c20f8928ec37..4edc65b29d6633 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DescriptionVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DescriptionVisitorTest.java @@ -1,7 +1,7 @@ package datahub.protobuf.visitors.dataset; import datahub.protobuf.model.ProtobufGraph; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DomainVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DomainVisitorTest.java index 0420953a647cbd..b3fa2c8fd081b0 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DomainVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DomainVisitorTest.java @@ -2,7 +2,7 @@ import com.linkedin.common.urn.Urn; import datahub.protobuf.model.ProtobufGraph; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/InstitutionalMemoryVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/InstitutionalMemoryVisitorTest.java index a313681c5a5a05..09fc0a37654364 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/InstitutionalMemoryVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/InstitutionalMemoryVisitorTest.java @@ -2,7 +2,7 @@ import com.linkedin.common.InstitutionalMemoryMetadata; import com.linkedin.common.url.Url; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/KafkaTopicPropertyVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/KafkaTopicPropertyVisitorTest.java index 84e7eb19f893b5..971500b5f43a2e 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/KafkaTopicPropertyVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/KafkaTopicPropertyVisitorTest.java @@ -2,7 +2,7 @@ import com.linkedin.data.template.StringMap; import com.linkedin.dataset.DatasetProperties; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java index cf2649e86dc43a..b087c683f9ffe4 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java @@ -6,7 +6,7 @@ import com.linkedin.common.OwnershipType; import com.linkedin.common.urn.Urn; import datahub.protobuf.model.ProtobufGraph; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/PropertyVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/PropertyVisitorTest.java index 2316416729befb..dc3647cdf34c80 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/PropertyVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/PropertyVisitorTest.java @@ -2,7 +2,7 @@ import com.linkedin.data.template.StringMap; import com.linkedin.dataset.DatasetProperties; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/TermAssociationVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/TermAssociationVisitorTest.java index 04fd52cf82e84e..c140a798ef6e6d 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/TermAssociationVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/TermAssociationVisitorTest.java @@ -2,7 +2,7 @@ import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.urn.GlossaryTermUrn; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/ProtobufExtensionFieldVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/ProtobufExtensionFieldVisitorTest.java index 0a1928310bfc27..57a8cf1d63cd26 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/ProtobufExtensionFieldVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/ProtobufExtensionFieldVisitorTest.java @@ -15,7 +15,7 @@ import com.linkedin.schema.StringType; import com.linkedin.util.Pair; import datahub.protobuf.ProtobufDataset; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URISyntaxException; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/SchemaFieldVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/SchemaFieldVisitorTest.java index 6c855e70d7f371..1da29b5320637a 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/SchemaFieldVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/field/SchemaFieldVisitorTest.java @@ -7,7 +7,7 @@ import com.linkedin.schema.UnionType; import com.linkedin.util.Pair; import datahub.protobuf.ProtobufDataset; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/tag/TagVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/tag/TagVisitorTest.java index 6fe1098f5e99ab..84ab1312a7d8a1 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/tag/TagVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/tag/TagVisitorTest.java @@ -3,7 +3,7 @@ import com.linkedin.tag.TagProperties; import datahub.protobuf.visitors.tags.TagVisitor; import datahub.event.MetadataChangeProposalWrapper; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index a2c643516dce64..ad54cf65243982 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -31,7 +31,7 @@ dependencies { api externalDependency.datastaxOssCore api externalDependency.datastaxOssQueryBuilder api externalDependency.elasticSearchRest - api externalDependency.elasticSearchTransport + api externalDependency.elasticSearchJava implementation externalDependency.javatuples api externalDependency.javaxValidation runtimeOnly externalDependency.jna @@ -64,6 +64,7 @@ dependencies { testImplementation externalDependency.testContainers testImplementation externalDependency.testContainersJunit testImplementation externalDependency.testContainersElasticsearch + testImplementation externalDependency.testContainersOpenSearch testImplementation externalDependency.testContainersCassandra testImplementation externalDependency.lombok testImplementation externalDependency.springBootTest @@ -101,14 +102,20 @@ dependencies { } test { - // https://docs.gradle.org/current/userguide/performance.html - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + doFirst { + // override, testng controlling parallelization + // increasing >1 will merely run all tests extra times + maxParallelForks = 1 + } + useTestNG() { + suites 'src/test/resources/testng.xml' + } testLogging.showStandardStreams = true testLogging.exceptionFormat = 'full' } tasks.withType(Test) { - enableAssertions = false + enableAssertions = false } project.compileJava { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index 8df7a9600ca94c..946931a54f4ec3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -45,15 +45,15 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; import static com.linkedin.metadata.graph.elastic.ElasticSearchGraphService.*; @@ -297,12 +297,12 @@ private List getLineageRelationships(@Nonnull List ent // Get search query for given list of edges and source urns @VisibleForTesting - static QueryBuilder getQueryForLineage( - @Nonnull List urns, - @Nonnull List lineageEdges, - @Nonnull GraphFilters graphFilters, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + public static QueryBuilder getQueryForLineage( + @Nonnull List urns, + @Nonnull List lineageEdges, + @Nonnull GraphFilters graphFilters, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis) { BoolQueryBuilder query = QueryBuilders.boolQuery(); if (lineageEdges.isEmpty()) { return query; @@ -361,10 +361,10 @@ static QueryBuilder getQueryForLineage( * physically stored inside the Graph Store. */ @VisibleForTesting - static void addEdgeToPaths( - @Nonnull final Map existingPaths, - @Nonnull final Urn parentUrn, - @Nonnull final Urn childUrn) { + public static void addEdgeToPaths( + @Nonnull final Map existingPaths, + @Nonnull final Urn parentUrn, + @Nonnull final Urn childUrn) { // Collect all full-paths to this child node. This is what will be returned. UrnArrayArray pathsToParent = existingPaths.get(parentUrn); if (pathsToParent != null && pathsToParent.size() > 0) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphWriteDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphWriteDAO.java index 8d2fcaa857541c..f8b0e8a291e7a2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphWriteDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphWriteDAO.java @@ -10,11 +10,11 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.reindex.BulkByScrollResponse; import static com.linkedin.metadata.graph.elastic.ESGraphQueryDAO.buildQuery; import static com.linkedin.metadata.graph.elastic.ElasticSearchGraphService.INDEX_NAME; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java index 346befca22559b..02e36af343b071 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java @@ -45,8 +45,8 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.QueryBuilders; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.index.query.QueryBuilders; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/TimeFilterUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/TimeFilterUtils.java index 66422c5997d17e..1df938f902e0fd 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/TimeFilterUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/TimeFilterUtils.java @@ -1,9 +1,9 @@ package com.linkedin.metadata.graph.elastic; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; import static com.linkedin.metadata.graph.elastic.ESGraphQueryDAO.*; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java index ea1f6cead80a9c..6985ceb00afd26 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java @@ -28,17 +28,17 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java index 402b579b13879b..dc30d4c80abc01 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java @@ -28,18 +28,18 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java index 6ef207dada4979..0836c569ed5d17 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java @@ -28,18 +28,18 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index 32adce458770d4..bf4dffe9e5fb8f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -27,7 +27,7 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponse; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index 14f67ddcbf3370..10c2fd725dca99 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -30,36 +30,36 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.config.RequestConfig; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.client.indices.GetIndexResponse; -import org.elasticsearch.client.indices.GetMappingsRequest; -import org.elasticsearch.client.indices.PutMappingRequest; -import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.tasks.TaskInfo; +import org.opensearch.OpenSearchException; +import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksRequest; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.GetAliasesResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.core.CountRequest; +import org.opensearch.client.indices.CreateIndexRequest; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.client.indices.GetIndexResponse; +import org.opensearch.client.indices.GetMappingsRequest; +import org.opensearch.client.indices.PutMappingRequest; +import org.opensearch.client.tasks.TaskSubmissionResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.reindex.ReindexRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.tasks.TaskInfo; @Slf4j @@ -117,7 +117,7 @@ public ESIndexBuilder(RestHighLevelClient searchClient, int numShards, int numRe RetryConfig config = RetryConfig.custom() .maxAttempts(Math.max(1, numRetries)) .waitDuration(Duration.ofSeconds(10)) - .retryOnException(e -> e instanceof ElasticsearchException) + .retryOnException(e -> e instanceof OpenSearchException) .failAfterMaxAttempts(true) .build(); @@ -153,7 +153,8 @@ public ReindexConfig buildReindexState(String indexName, Map map Settings currentSettings = _searchClient.indices() .getSettings(new GetSettingsRequest().indices(indexName), RequestOptions.DEFAULT) .getIndexToSettings() - .valuesIt() + .values() + .iterator() .next(); builder.currentSettings(currentSettings); @@ -170,6 +171,15 @@ public ReindexConfig buildReindexState(String indexName, Map map return builder.build(); } + /** + * Builds index with given name, mappings and settings + * Deprecated: Use the `buildIndex(ReindexConfig indexState) to enforce conventions via ReindexConfig class + * earlier in the process. + * @param indexName index name + * @param mappings ES mappings + * @param settings ES settings + * @throws IOException ES error + */ @Deprecated public void buildIndex(String indexName, Map mappings, Map settings) throws IOException { buildIndex(buildReindexState(indexName, mappings, settings)); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java index a0c0bd85c04c6e..4f5f2926d3da03 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java @@ -10,7 +10,7 @@ import lombok.Getter; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.common.settings.Settings; +import org.opensearch.common.settings.Settings; import java.util.List; import java.util.Map; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 3cb3c441afd68e..5fd0a80d23c505 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -38,21 +38,21 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortOrder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.terms.IncludeExclude; +import org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; import static com.linkedin.metadata.utils.SearchUtil.filterSoftDeletedByDefault; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index f3864d99ba5e98..cbaf70ca22617d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -35,20 +35,20 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.core.CountRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.search.SearchModule; +import org.opensearch.search.builder.SearchSourceBuilder; import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; @@ -63,7 +63,7 @@ public class ESSearchDAO { private static final NamedXContentRegistry X_CONTENT_REGISTRY; static { - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); } @@ -137,7 +137,7 @@ private AggregationMetadata transformAggregationMetadata(@Nonnull AggregationMet } @VisibleForTesting - SearchResult transformIndexIntoEntityName(SearchResult result) { + public SearchResult transformIndexIntoEntityName(SearchResult result) { return result.setMetadata(result.getMetadata().setAggregations(transformIndexIntoEntityName(result.getMetadata().getAggregations()))); } private ScrollResult transformIndexIntoEntityName(ScrollResult result) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index d95bbcf893628c..e2bdea84eda0ea 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -11,8 +11,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; import static com.linkedin.metadata.utils.SearchUtil.*; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index f4be46e58f3b86..bba3a9fa4232d4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -24,15 +24,15 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MultiMatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/PITAwareSearchRequest.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/PITAwareSearchRequest.java index c0b1ac028e9d48..79c00fc7cdd20e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/PITAwareSearchRequest.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/PITAwareSearchRequest.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search.elasticsearch.query.request; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.support.IndicesOptions; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.IndicesOptions; public class PITAwareSearchRequest extends SearchRequest { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index b01c736ec23ae5..ce88f31449c35d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -35,24 +35,24 @@ import com.linkedin.metadata.search.utils.ESUtils; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.common.lucene.search.function.CombineFunction; -import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; -import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.Operator; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.index.query.SimpleQueryStringBuilder; -import org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; -import org.elasticsearch.search.SearchModule; +import org.opensearch.common.lucene.search.function.CombineFunction; +import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; +import org.opensearch.common.lucene.search.function.FunctionScoreQuery; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.opensearch.index.query.functionscore.ScoreFunctionBuilders; +import org.opensearch.search.SearchModule; import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; @@ -69,7 +69,7 @@ public class SearchQueryBuilder { } private static final NamedXContentRegistry X_CONTENT_REGISTRY; static { - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); } @@ -135,7 +135,7 @@ private QueryBuilder buildInternalQuery(@Nullable QueryConfiguration customQuery * @return A set of SearchFieldConfigs containing the searchable fields from the input entities. */ @VisibleForTesting - Set getStandardFields(@Nonnull Collection entitySpecs) { + public Set getStandardFields(@Nonnull Collection entitySpecs) { Set fields = new HashSet<>(); // Always present final float urnBoost = Float.parseFloat((String) PRIMARY_URN_SEARCH_PROPERTIES.get("boostScore")); @@ -168,7 +168,7 @@ Set getStandardFields(@Nonnull Collection entityS } @VisibleForTesting - Set getFieldsFromEntitySpec(EntitySpec entitySpec) { + public Set getFieldsFromEntitySpec(EntitySpec entitySpec) { Set fields = new HashSet<>(); List searchableFieldSpecs = entitySpec.getSearchableFieldSpecs(); for (SearchableFieldSpec fieldSpec : searchableFieldSpecs) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index dbd933d59d7f36..5fcc10b7af5cfa 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -54,23 +54,23 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.text.Text; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; -import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.text.Text; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightField; +import org.opensearch.search.suggest.term.TermSuggestion; import static com.linkedin.metadata.search.utils.ESUtils.NAME_SUGGESTION; import static com.linkedin.metadata.search.utils.ESUtils.toFacetField; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/BulkListener.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/BulkListener.java index 297453bdce517e..be64df3179a9d2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/BulkListener.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/BulkListener.java @@ -2,11 +2,11 @@ import com.linkedin.metadata.utils.metrics.MetricUtils; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.support.WriteRequest; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.bulk.BulkProcessor; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.support.WriteRequest; import java.util.Arrays; import java.util.HashMap; @@ -76,7 +76,7 @@ private static String buildMetricName(DocWriteRequest.OpType opType, String stat public static String buildBulkRequestSummary(BulkRequest request) { return request.requests().stream().map(req -> String.format( "Failed to perform bulk request: index [%s], optype: [%s], type [%s], id [%s]", - req.index(), req.opType(), req.type(), req.id()) + req.index(), req.opType(), req.opType(), req.id()) ).collect(Collectors.joining(";")); } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java index a7ece47a7f5d6d..a1e5b363d8a78b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java @@ -7,19 +7,19 @@ import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.bulk.BackoffPolicy; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.bulk.BackoffPolicy; +import org.opensearch.action.bulk.BulkProcessor; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.tasks.TaskSubmissionResponse; +import org.opensearch.common.Nullable; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.reindex.BulkByScrollResponse; +import org.opensearch.index.reindex.DeleteByQueryRequest; import java.io.Closeable; import java.io.IOException; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java index 1a63f2d4d03122..edcdf5654028ce 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java @@ -6,15 +6,15 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.client.indices.GetIndexResponse; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.script.Script; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.client.indices.GetIndexResponse; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.script.Script; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 12c081a5c25a6a..9a7d9a1b4c4207 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -16,21 +16,21 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.RequestOptions; +import org.opensearch.client.RequestOptions; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.PointInTimeBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestBuilders; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.ScoreSortBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.search.suggest.SuggestBuilder; +import org.opensearch.search.suggest.SuggestBuilders; +import org.opensearch.search.suggest.SuggestionBuilder; +import org.opensearch.search.suggest.term.TermSuggestionBuilder; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.KEYWORD_FIELDS; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.PATH_HIERARCHY_FIELDS; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ESSystemMetadataDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ESSystemMetadataDAO.java index c7e8d0940c5307..5eb03eb23d01a7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ESSystemMetadataDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ESSystemMetadataDAO.java @@ -13,26 +13,26 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.tasks.GetTaskRequest; -import org.elasticsearch.client.tasks.GetTaskResponse; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.BucketSortPipelineAggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortOrder; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.tasks.GetTaskRequest; +import org.opensearch.client.tasks.GetTaskResponse; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.reindex.BulkByScrollResponse; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.PipelineAggregatorBuilders; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.pipeline.BucketSortPipelineAggregationBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; import static com.linkedin.metadata.systemmetadata.ElasticSearchSystemMetadataService.INDEX_NAME; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java index 3fcb62424853a9..dd8e19861ccd21 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java @@ -31,14 +31,14 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.tasks.GetTaskResponse; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.ParsedMax; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.tasks.GetTaskResponse; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; +import org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.aggregations.metrics.ParsedMax; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java index 01fe41718d7f02..43ba87f474d6aa 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java @@ -49,26 +49,26 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.client.core.CountResponse; -import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.core.CountRequest; +import org.opensearch.client.core.CountResponse; +import org.opensearch.client.tasks.TaskSubmissionResponse; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; import static com.linkedin.metadata.Constants.*; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java index 6c5dbf2582c05a..b0751a9c6f9ea5 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java @@ -17,7 +17,7 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilder; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java index 5389d602ae5c1a..316d25d1f37f4d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java @@ -30,24 +30,24 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.metrics.ParsedCardinality; -import org.elasticsearch.search.aggregations.metrics.ParsedSum; -import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.ParsedBucketMetricValue; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.PipelineAggregatorBuilders; +import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.opensearch.search.aggregations.metrics.ParsedCardinality; +import org.opensearch.search.aggregations.metrics.ParsedSum; +import org.opensearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; +import org.opensearch.search.aggregations.pipeline.ParsedBucketMetricValue; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java b/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java deleted file mode 100644 index 327447341badf6..00000000000000 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.linkedin.metadata; - -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import com.linkedin.metadata.config.search.CustomConfiguration; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; -import com.linkedin.metadata.config.search.ExactMatchConfiguration; -import com.linkedin.metadata.config.search.PartialConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.WordGramConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; -import com.linkedin.metadata.models.registry.ConfigEntityRegistry; -import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.models.registry.EntityRegistryException; -import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; -import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; -import com.linkedin.metadata.version.GitVersion; -import java.util.Optional; -import org.apache.http.HttpHost; -import org.apache.http.impl.nio.reactor.IOReactorConfig; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Scope; -import org.testcontainers.elasticsearch.ElasticsearchContainer; - -import javax.annotation.Nonnull; - -import java.util.Map; - - -@TestConfiguration -public class ESTestConfiguration { - private static final int HTTP_PORT = 9200; - public static final int REFRESH_INTERVAL_SECONDS = 5; - - public static void syncAfterWrite(ESBulkProcessor bulkProcessor) throws InterruptedException { - bulkProcessor.flush(); - Thread.sleep(1000); - } - - @Bean - public SearchConfiguration searchConfiguration() { - SearchConfiguration searchConfiguration = new SearchConfiguration(); - searchConfiguration.setMaxTermBucketSize(20); - - ExactMatchConfiguration exactMatchConfiguration = new ExactMatchConfiguration(); - exactMatchConfiguration.setExclusive(false); - exactMatchConfiguration.setExactFactor(10.0f); - exactMatchConfiguration.setWithPrefix(true); - exactMatchConfiguration.setPrefixFactor(6.0f); - exactMatchConfiguration.setCaseSensitivityFactor(0.7f); - exactMatchConfiguration.setEnableStructured(true); - - WordGramConfiguration wordGramConfiguration = new WordGramConfiguration(); - wordGramConfiguration.setTwoGramFactor(1.2f); - wordGramConfiguration.setThreeGramFactor(1.5f); - wordGramConfiguration.setFourGramFactor(1.8f); - - PartialConfiguration partialConfiguration = new PartialConfiguration(); - partialConfiguration.setFactor(0.4f); - partialConfiguration.setUrnFactor(0.5f); - - searchConfiguration.setExactMatch(exactMatchConfiguration); - searchConfiguration.setWordGram(wordGramConfiguration); - searchConfiguration.setPartial(partialConfiguration); - return searchConfiguration; - } - - @Bean - public CustomSearchConfiguration customSearchConfiguration() throws Exception { - CustomConfiguration customConfiguration = new CustomConfiguration(); - customConfiguration.setEnabled(true); - customConfiguration.setFile("search_config_builder_test.yml"); - return customConfiguration.resolve(new YAMLMapper()); - } - - @Scope("singleton") - @Bean(name = "testElasticsearchContainer") - @Nonnull - public ElasticsearchContainer elasticsearchContainer() { - ESTestUtils.ES_CONTAINER.start(); - return ESTestUtils.ES_CONTAINER; - } - - @Primary - @Scope("singleton") - @Bean(name = "elasticSearchRestHighLevelClient") - @Nonnull - public RestHighLevelClient getElasticsearchClient(@Qualifier("testElasticsearchContainer") ElasticsearchContainer esContainer) { - // A helper method to create an ElasticseachContainer defaulting to the current image and version, with the ability - // within firewalled environments to override with an environment variable to point to the offline repository. - // A helper method to construct a standard rest client for Elastic search. - final RestClientBuilder builder = - RestClient.builder(new HttpHost( - "localhost", - esContainer.getMappedPort(HTTP_PORT), "http") - ).setHttpClientConfigCallback(httpAsyncClientBuilder -> - httpAsyncClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build())); - - builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder. - setConnectionRequestTimeout(30000)); - - return new RestHighLevelClient(builder); - } - - /* - Cannot use the factory class without circular dependencies - */ - @Primary - @Bean(name = "elasticSearchBulkProcessor") - @Nonnull - public ESBulkProcessor getBulkProcessor(@Qualifier("elasticSearchRestHighLevelClient") RestHighLevelClient searchClient) { - return ESBulkProcessor.builder(searchClient) - .async(true) - /* - * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful - * to present a consistent view to for indices with very low traffic. And it is wonderful for tests! - */ - .writeRequestRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .bulkRequestsLimit(10000) - .bulkFlushPeriod(REFRESH_INTERVAL_SECONDS - 1) - .retryInterval(1L) - .numRetries(1) - .build(); - } - - @Primary - @Bean(name = "elasticSearchIndexBuilder") - @Nonnull - protected ESIndexBuilder getIndexBuilder(@Qualifier("elasticSearchRestHighLevelClient") RestHighLevelClient searchClient) { - GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); - return new ESIndexBuilder(searchClient, 1, 1, 3, 1, Map.of(), - false, false, - new ElasticSearchConfiguration(), gitVersion); - } - - @Bean(name = "entityRegistry") - public EntityRegistry entityRegistry() throws EntityRegistryException { - return new ConfigEntityRegistry( - ESTestConfiguration.class.getClassLoader().getResourceAsStream("entity-registry.yml")); - } - - @Bean(name = "longTailEntityRegistry") - public EntityRegistry longTailEntityRegistry() throws EntityRegistryException { - return new ConfigEntityRegistry( - ESTestConfiguration.class.getClassLoader().getResourceAsStream("entity-registry.yml")); - } -} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/EbeanTestUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/EbeanTestUtils.java index 180166e963fca3..a7308f079fc200 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/EbeanTestUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/EbeanTestUtils.java @@ -2,7 +2,7 @@ import io.ebean.Database; import io.ebean.DatabaseFactory; -import io.ebean.config.ServerConfig; +import io.ebean.config.DatabaseConfig; import io.ebean.datasource.DataSourceConfig; import javax.annotation.Nonnull; @@ -14,22 +14,33 @@ private EbeanTestUtils() { @Nonnull public static Database createTestServer() { - return DatabaseFactory.create(createTestingH2ServerConfig()); + final DatabaseConfig config = createTestingH2ServerConfig(); + Database server; + // designed for re-use across tests + try { + server = DatabaseFactory.create(config); + } catch(Exception e) { + config.setDdlRun(false); + server = DatabaseFactory.create(config); + } + server.truncate("metadata_aspect_v2"); + return server; } @Nonnull - private static ServerConfig createTestingH2ServerConfig() { + private static DatabaseConfig createTestingH2ServerConfig() { DataSourceConfig dataSourceConfig = new DataSourceConfig(); dataSourceConfig.setUsername("tester"); dataSourceConfig.setPassword(""); dataSourceConfig.setUrl("jdbc:h2:mem:test;IGNORECASE=TRUE;mode=mysql;"); dataSourceConfig.setDriver("org.h2.Driver"); - ServerConfig serverConfig = new ServerConfig(); + DatabaseConfig serverConfig = new DatabaseConfig(); serverConfig.setName("gma"); serverConfig.setDataSourceConfig(dataSourceConfig); serverConfig.setDdlGenerate(true); serverConfig.setDdlRun(true); + serverConfig.setDdlCreateOnly(true); return serverConfig; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/BulkListenerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/BulkListenerTest.java index 154131ceb6fee5..10a73cbe532a23 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/BulkListenerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/BulkListenerTest.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.elasticsearch.update; import com.linkedin.metadata.search.elasticsearch.update.BulkListener; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.support.WriteRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.support.WriteRequest; import org.mockito.Mockito; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/ESBulkProcessorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/ESBulkProcessorTest.java index 5c882e5158f903..2d84c9f3444de6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/ESBulkProcessorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/elasticsearch/update/ESBulkProcessorTest.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.elasticsearch.update; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.mockito.Mockito; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 90f9baa4ca4c2c..702bc14a86ac39 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -22,7 +22,7 @@ import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; -import io.datahub.test.DataGenerator; +import io.datahubproject.test.DataGenerator; import io.ebean.Database; import io.ebean.Transaction; import io.ebean.TxScope; @@ -62,6 +62,12 @@ public EbeanEntityServiceTest() throws EntityRegistryException { @BeforeMethod public void setupTest() { Database server = EbeanTestUtils.createTestServer(); + + Long count = server.sqlQuery("select count(*) from metadata_aspect_v2") + .mapToScalar(Long.class) + .findOne(); + assertEquals(count, 0, "Unexpected data present in test database."); + _mockProducer = mock(EventProducer.class); _aspectDao = new EbeanAspectDao(server); @@ -239,6 +245,7 @@ public void testNestedTransactions() throws AssertionError { System.out.println("done"); } + @Test public void dataGeneratorThreadingTest() { DataGenerator dataGenerator = new DataGenerator(_entityServiceImpl); @@ -262,7 +269,7 @@ public void dataGeneratorThreadingTest() { * This test is designed to detect multi-threading persistence exceptions like duplicate key, * exceptions that exceed retry limits or unnecessary versions. */ - @Test + @Test // ensure same thread as h2 public void multiThreadingTest() { DataGenerator dataGenerator = new DataGenerator(_entityServiceImpl); Database server = ((EbeanAspectDao) _entityServiceImpl._aspectDao).getServer(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java similarity index 98% rename from metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAOTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java index 3ba2c858fb1a3c..baed3ade0d2077 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.graph.elastic; +package com.linkedin.metadata.graph.search; import com.google.common.collect.ImmutableList; import com.google.common.io.Resources; @@ -8,6 +8,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.graph.GraphFilters; +import com.linkedin.metadata.graph.elastic.ESGraphQueryDAO; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.query.filter.RelationshipDirection; import java.net.URL; @@ -16,7 +17,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.elasticsearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java similarity index 93% rename from metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java index 1717e466359d3d..0ce43c9d315716 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java @@ -1,12 +1,11 @@ -package com.linkedin.metadata.graph.elastic; +package com.linkedin.metadata.graph.search; -import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.common.FabricType; import com.linkedin.common.urn.DataPlatformUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; -import com.linkedin.metadata.ESTestConfiguration; +import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; @@ -14,6 +13,9 @@ import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.graph.RelatedEntitiesResult; import com.linkedin.metadata.graph.RelatedEntity; +import com.linkedin.metadata.graph.elastic.ESGraphQueryDAO; +import com.linkedin.metadata.graph.elastic.ESGraphWriteDAO; +import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.query.filter.Filter; @@ -23,18 +25,17 @@ import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; -import java.util.Arrays; -import java.util.Collections; -import org.elasticsearch.client.RestHighLevelClient; +import io.datahubproject.test.search.SearchTestUtils; import org.junit.Assert; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Import; +import org.opensearch.client.RestHighLevelClient; import org.testng.SkipException; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import javax.annotation.Nonnull; +import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -43,15 +44,16 @@ import static com.linkedin.metadata.search.utils.QueryUtils.*; import static org.testng.Assert.assertEquals; -@Import(ESTestConfiguration.class) -public class ElasticSearchGraphServiceTest extends GraphServiceTestBase { +abstract public class SearchGraphServiceTestBase extends GraphServiceTestBase { - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private ESBulkProcessor _bulkProcessor; - @Autowired - private ESIndexBuilder _esIndexBuilder; + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); + + @Nonnull + abstract protected ESBulkProcessor getBulkProcessor(); + + @Nonnull + abstract protected ESIndexBuilder getIndexBuilder(); private final IndexConvention _indexConvention = new IndexConventionImpl(null); private final String _indexName = _indexConvention.getIndexName(INDEX_NAME); @@ -74,10 +76,10 @@ public void wipe() throws Exception { @Nonnull private ElasticSearchGraphService buildService() { LineageRegistry lineageRegistry = new LineageRegistry(SnapshotEntityRegistry.getInstance()); - ESGraphQueryDAO readDAO = new ESGraphQueryDAO(_searchClient, lineageRegistry, _indexConvention, GraphQueryConfiguration.testDefaults); - ESGraphWriteDAO writeDAO = new ESGraphWriteDAO(_indexConvention, _bulkProcessor, 1); - return new ElasticSearchGraphService(lineageRegistry, _bulkProcessor, _indexConvention, writeDAO, readDAO, - _esIndexBuilder); + ESGraphQueryDAO readDAO = new ESGraphQueryDAO(getSearchClient(), lineageRegistry, _indexConvention, GraphQueryConfiguration.testDefaults); + ESGraphWriteDAO writeDAO = new ESGraphWriteDAO(_indexConvention, getBulkProcessor(), 1); + return new ElasticSearchGraphService(lineageRegistry, getBulkProcessor(), _indexConvention, writeDAO, readDAO, + getIndexBuilder()); } @Override @@ -88,7 +90,7 @@ protected GraphService getGraphService() { @Override protected void syncAfterWrite() throws Exception { - ESTestConfiguration.syncAfterWrite(_bulkProcessor); + SearchTestUtils.syncAfterWrite(getBulkProcessor()); } @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/TimeFilterUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/TimeFilterUtilsTest.java similarity index 82% rename from metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/TimeFilterUtilsTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/graph/search/TimeFilterUtilsTest.java index 988a7ccc707413..989f9ae197239e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/TimeFilterUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/TimeFilterUtilsTest.java @@ -1,9 +1,11 @@ -package com.linkedin.metadata.graph.elastic; +package com.linkedin.metadata.graph.search; import com.google.common.io.Resources; import java.net.URL; import java.nio.charset.StandardCharsets; -import org.elasticsearch.index.query.QueryBuilder; + +import com.linkedin.metadata.graph.elastic.TimeFilterUtils; +import org.opensearch.index.query.QueryBuilder; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java new file mode 100644 index 00000000000000..7b550311bf8238 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java @@ -0,0 +1,49 @@ +package com.linkedin.metadata.graph.search.elasticsearch; + +import com.linkedin.metadata.graph.search.SearchGraphServiceTestBase; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchSuite; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; + +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) +public class SearchGraphServiceElasticSearchTest extends SearchGraphServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } + +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java new file mode 100644 index 00000000000000..eabfb523fb9108 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java @@ -0,0 +1,48 @@ +package com.linkedin.metadata.graph.search.opensearch; + +import com.linkedin.metadata.graph.search.SearchGraphServiceTestBase; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import com.linkedin.metadata.search.opensearch.OpenSearchSuite; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) +public class SearchGraphServiceOpenSearchTest extends SearchGraphServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } + +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java similarity index 94% rename from metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index faff9f780e31c9..461a146022446c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -1,8 +1,5 @@ package com.linkedin.metadata.search; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -16,8 +13,10 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; import com.linkedin.data.template.LongMap; -import com.linkedin.metadata.ESTestConfiguration; import com.linkedin.metadata.TestEntityUtil; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; +import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; @@ -47,47 +46,60 @@ import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.elasticsearch.client.RestHighLevelClient; import org.junit.Assert; import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; +import org.opensearch.client.RestHighLevelClient; import org.springframework.cache.CacheManager; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; -import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestConfiguration.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; -import static org.testng.Assert.*; - -@Import(ESTestConfiguration.class) -public class LineageSearchServiceTest extends AbstractTestNGSpringContextTests { - - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private ESBulkProcessor _bulkProcessor; - @Autowired - private ESIndexBuilder _esIndexBuilder; - @Autowired - private SearchConfiguration _searchConfiguration; - @Autowired - private CustomSearchConfiguration _customSearchConfiguration; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH; +import static io.datahubproject.test.search.SearchTestUtils.syncAfterWrite; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +abstract public class LineageServiceTestBase extends AbstractTestNGSpringContextTests { + + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); + + @Nonnull + abstract protected ESBulkProcessor getBulkProcessor(); + + @Nonnull + abstract protected ESIndexBuilder getIndexBuilder(); + + @Nonnull + abstract protected SearchConfiguration getSearchConfiguration(); + + @Nonnull + abstract protected CustomSearchConfiguration getCustomSearchConfiguration(); private EntityRegistry _entityRegistry; private IndexConvention _indexConvention; @@ -142,18 +154,18 @@ private void resetService(boolean withCache, boolean withLightingCache) { public void wipe() throws Exception { _elasticSearchService.clear(); clearCache(false); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); } @Nonnull private ElasticSearchService buildEntitySearchService() { EntityIndexBuilders indexBuilders = - new EntityIndexBuilders(_esIndexBuilder, _entityRegistry, + new EntityIndexBuilders(getIndexBuilder(), _entityRegistry, _indexConvention, _settingsBuilder); - ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, _searchClient, _indexConvention, false, - ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, null); - ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, _searchClient, _indexConvention, _searchConfiguration, _customSearchConfiguration); - ESWriteDAO writeDAO = new ESWriteDAO(_entityRegistry, _searchClient, _indexConvention, _bulkProcessor, 1); + ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, getSearchClient(), _indexConvention, false, + ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, getSearchConfiguration(), null); + ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, getSearchClient(), _indexConvention, getSearchConfiguration(), getCustomSearchConfiguration()); + ESWriteDAO writeDAO = new ESWriteDAO(_entityRegistry, getSearchClient(), _indexConvention, getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } @@ -198,7 +210,7 @@ public void testSearchService() throws Exception { document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride")); document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt(), eq(null), eq(null))).thenReturn(mockResult(Collections.emptyList())); @@ -232,7 +244,7 @@ public void testSearchService() throws Exception { document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride2")); document2.set("browsePaths", JsonNodeFactory.instance.textNode("/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = searchAcrossLineage(null, TEST1); assertEquals(searchResult.getNumEntities().intValue(), 1); @@ -306,7 +318,7 @@ public void testSearchService() throws Exception { // Cleanup _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt())).thenReturn( @@ -350,7 +362,7 @@ public void testScrollAcrossLineage() throws Exception { document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride")); document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt(), eq(null), eq(null))).thenReturn(mockResult(Collections.emptyList())); @@ -383,7 +395,7 @@ public void testScrollAcrossLineage() throws Exception { // Cleanup _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt())).thenReturn( @@ -424,7 +436,7 @@ public void testLightningSearchService() throws Exception { document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride")); document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt(), eq(null), eq(null))).thenReturn(mockResult(Collections.emptyList())); @@ -461,7 +473,7 @@ public void testLightningSearchService() throws Exception { document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride2")); document2.set("browsePaths", JsonNodeFactory.instance.textNode("/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = searchAcrossLineage(null, testStar); assertEquals(searchResult.getNumEntities().intValue(), 1); @@ -616,7 +628,7 @@ public void testLightningSearchService() throws Exception { // Cleanup _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt())).thenReturn( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java similarity index 92% rename from metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java index ad836664d7f6dc..c0144d36843f57 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java @@ -1,7 +1,5 @@ package com.linkedin.metadata.search; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -9,7 +7,8 @@ import com.linkedin.common.urn.TestEntityUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; -import com.linkedin.metadata.ESTestConfiguration; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; @@ -33,11 +32,9 @@ import com.linkedin.metadata.search.ranker.SimpleRanker; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; +import org.opensearch.client.RestHighLevelClient; import org.springframework.cache.CacheManager; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; -import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -45,23 +42,28 @@ import javax.annotation.Nonnull; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestConfiguration.syncAfterWrite; +import static com.linkedin.metadata.Constants.ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH; +import static io.datahubproject.test.search.SearchTestUtils.syncAfterWrite; import static org.testng.Assert.assertEquals; -@Import(ESTestConfiguration.class) -public class SearchServiceTest extends AbstractTestNGSpringContextTests { - - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private ESBulkProcessor _bulkProcessor; - @Autowired - private ESIndexBuilder _esIndexBuilder; - @Autowired - private SearchConfiguration _searchConfiguration; - @Autowired - private CustomSearchConfiguration _customSearchConfiguration; + +abstract public class SearchServiceTestBase extends AbstractTestNGSpringContextTests { + + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); + + @Nonnull + abstract protected ESBulkProcessor getBulkProcessor(); + + @Nonnull + abstract protected ESIndexBuilder getIndexBuilder(); + + @Nonnull + abstract protected SearchConfiguration getSearchConfiguration(); + + @Nonnull + abstract protected CustomSearchConfiguration getCustomSearchConfiguration(); + private EntityRegistry _entityRegistry; private IndexConvention _indexConvention; private SettingsBuilder _settingsBuilder; @@ -100,19 +102,19 @@ private void resetSearchService() { @BeforeMethod public void wipe() throws Exception { _elasticSearchService.clear(); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); } @Nonnull private ElasticSearchService buildEntitySearchService() { EntityIndexBuilders indexBuilders = - new EntityIndexBuilders(_esIndexBuilder, _entityRegistry, + new EntityIndexBuilders(getIndexBuilder(), _entityRegistry, _indexConvention, _settingsBuilder); - ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, _searchClient, _indexConvention, false, - ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, null); - ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, _searchClient, _indexConvention, _searchConfiguration, _customSearchConfiguration); - ESWriteDAO writeDAO = new ESWriteDAO(_entityRegistry, _searchClient, _indexConvention, - _bulkProcessor, 1); + ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, getSearchClient(), _indexConvention, false, + ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, getSearchConfiguration(), null); + ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, getSearchClient(), _indexConvention, getSearchConfiguration(), getCustomSearchConfiguration()); + ESWriteDAO writeDAO = new ESWriteDAO(_entityRegistry, getSearchClient(), _indexConvention, + getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } @@ -139,7 +141,7 @@ public void testSearchService() throws Exception { document.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride")); document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", null, null, 0, 10, new SearchFlags().setFulltext(true)); @@ -154,7 +156,7 @@ public void testSearchService() throws Exception { document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride2")); document2.set("browsePaths", JsonNodeFactory.instance.textNode("/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "'test2'", null, null, 0, 10, new SearchFlags().setFulltext(true)); @@ -167,7 +169,7 @@ public void testSearchService() throws Exception { _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "'test2'", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -233,7 +235,7 @@ public void testAdvancedSearchOr() throws Exception { document3.set("platform", JsonNodeFactory.instance.textNode("snowflake")); _elasticSearchService.upsertDocument(ENTITY_NAME, document3.toString(), urn3.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", filterWithCondition, null, 0, 10, new SearchFlags().setFulltext(true)); @@ -304,7 +306,7 @@ public void testAdvancedSearchSoftDelete() throws Exception { document.set("removed", JsonNodeFactory.instance.booleanNode(false)); _elasticSearchService.upsertDocument(ENTITY_NAME, document3.toString(), urn3.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", filterWithCondition, null, 0, 10, new SearchFlags().setFulltext(true)); @@ -369,7 +371,7 @@ public void testAdvancedSearchNegated() throws Exception { document.set("removed", JsonNodeFactory.instance.booleanNode(false)); _elasticSearchService.upsertDocument(ENTITY_NAME, document3.toString(), urn3.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _searchService.searchAcrossEntities(ImmutableList.of(), "test", filterWithCondition, null, 0, 10, new SearchFlags().setFulltext(true)); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java similarity index 86% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java index 9a6d2dc6fc1fa5..d358c03c612d09 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java @@ -1,19 +1,18 @@ -package com.linkedin.metadata.search.elasticsearch; +package com.linkedin.metadata.search; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.urn.TestEntityUrn; import com.linkedin.common.urn.Urn; -import com.linkedin.metadata.ESTestConfiguration; import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.query.SearchFlags; -import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder; @@ -23,10 +22,7 @@ import com.linkedin.metadata.search.elasticsearch.update.ESWriteDAO; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; -import java.util.List; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Import; +import org.opensearch.client.RestHighLevelClient; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.annotations.BeforeClass; @@ -34,24 +30,28 @@ import org.testng.annotations.Test; import javax.annotation.Nonnull; +import java.util.List; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestConfiguration.syncAfterWrite; +import static com.linkedin.metadata.Constants.ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH; +import static io.datahubproject.test.search.SearchTestUtils.syncAfterWrite; import static org.testng.Assert.assertEquals; -@Import(ESTestConfiguration.class) -public class ElasticSearchServiceTest extends AbstractTestNGSpringContextTests { +abstract public class TestEntityTestBase extends AbstractTestNGSpringContextTests { - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private ESBulkProcessor _bulkProcessor; - @Autowired - private ESIndexBuilder _esIndexBuilder; - @Autowired - private SearchConfiguration _searchConfiguration; - @Autowired - private CustomSearchConfiguration _customSearchConfiguration; + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); + + @Nonnull + abstract protected ESBulkProcessor getBulkProcessor(); + + @Nonnull + abstract protected ESIndexBuilder getIndexBuilder(); + + @Nonnull + abstract protected SearchConfiguration getSearchConfiguration(); + + @Nonnull + abstract protected CustomSearchConfiguration getCustomSearchConfiguration(); private EntityRegistry _entityRegistry; private IndexConvention _indexConvention; @@ -83,12 +83,12 @@ public void wipe() throws Exception { @Nonnull private ElasticSearchService buildService() { EntityIndexBuilders indexBuilders = - new EntityIndexBuilders(_esIndexBuilder, _entityRegistry, _indexConvention, _settingsBuilder); - ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, _searchClient, _indexConvention, false, - ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, null); - ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, _searchClient, _indexConvention, _searchConfiguration, _customSearchConfiguration); + new EntityIndexBuilders(getIndexBuilder(), _entityRegistry, _indexConvention, _settingsBuilder); + ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, getSearchClient(), _indexConvention, false, + ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, getSearchConfiguration(), null); + ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, getSearchClient(), _indexConvention, getSearchConfiguration(), getCustomSearchConfiguration()); ESWriteDAO writeDAO = - new ESWriteDAO(_entityRegistry, _searchClient, _indexConvention, _bulkProcessor, 1); + new ESWriteDAO(_entityRegistry, getSearchClient(), _indexConvention, getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } @@ -109,7 +109,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c")); document.set("foreignKey", JsonNodeFactory.instance.textNode("urn:li:tag:Node.Value")); _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); @@ -134,7 +134,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride2")); document2.set("browsePaths", JsonNodeFactory.instance.textNode("/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); @@ -152,7 +152,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 0); browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); @@ -174,7 +174,7 @@ public void testElasticSearchServiceFulltext() throws Exception { document.set("browsePaths", JsonNodeFactory.instance.textNode("/a/b/c")); document.set("foreignKey", JsonNodeFactory.instance.textNode("urn:li:tag:Node.Value")); _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 1); @@ -191,7 +191,7 @@ public void testElasticSearchServiceFulltext() throws Exception { document2.set("textFieldOverride", JsonNodeFactory.instance.textNode("textFieldOverride2")); document2.set("browsePaths", JsonNodeFactory.instance.textNode("/b/c")); _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 1); @@ -203,7 +203,7 @@ public void testElasticSearchServiceFulltext() throws Exception { _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 0); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchSuite.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchSuite.java new file mode 100644 index 00000000000000..750423a024dcc3 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchSuite.java @@ -0,0 +1,32 @@ +package com.linkedin.metadata.search.elasticsearch; + +import io.datahubproject.test.search.ElasticsearchTestContainer; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testcontainers.containers.GenericContainer; +import org.testng.annotations.AfterSuite; + + +@TestConfiguration +public class ElasticSearchSuite extends AbstractTestNGSpringContextTests { + + private static final ElasticsearchTestContainer ELASTICSEARCH_TEST_CONTAINER; + private static GenericContainer container; + static { + ELASTICSEARCH_TEST_CONTAINER = new ElasticsearchTestContainer(); + } + + @AfterSuite + public void after() { + ELASTICSEARCH_TEST_CONTAINER.stopContainer(); + } + + @Bean(name = "testSearchContainer") + public GenericContainer testSearchContainer() { + if (container == null) { + container = ELASTICSEARCH_TEST_CONTAINER.startContainer(); + } + return container; + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java new file mode 100644 index 00000000000000..cfacd4c15409a5 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java @@ -0,0 +1,44 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.search.fixtures.GoldenTestBase; +import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.SearchService; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + +@Import({ElasticSearchSuite.class, SampleDataFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class GoldenElasticSearchTest extends GoldenTestBase { + + @Autowired + @Qualifier("longTailSearchService") + protected SearchService searchService; + + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry entityRegistry; + + + @NotNull + @Override + protected EntityRegistry getEntityRegistry() { + return entityRegistry; + } + + @NotNull + @Override + protected SearchService getSearchService() { + return searchService; + } + + @Test + public void initTest() { + assertNotNull(searchService); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java new file mode 100644 index 00000000000000..20f4ee52f0e621 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java @@ -0,0 +1,30 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.search.indexbuilder.IndexBuilderTestBase; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + + +@Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) +public class IndexBuilderElasticSearchTest extends IndexBuilderTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @Test + public void initTest() { + assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java new file mode 100644 index 00000000000000..0cb49bc5554216 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java @@ -0,0 +1,43 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.search.fixtures.LineageDataFixtureTestBase; +import io.datahubproject.test.fixtures.search.SearchLineageFixtureConfiguration; +import com.linkedin.metadata.search.LineageSearchService; +import com.linkedin.metadata.search.SearchService; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + + +@Import({ElasticSearchSuite.class, SearchLineageFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class LineageDataFixtureElasticSearchTest extends LineageDataFixtureTestBase { + + @Autowired + @Qualifier("searchLineageSearchService") + protected SearchService searchService; + + @Autowired + @Qualifier("searchLineageLineageSearchService") + protected LineageSearchService lineageService; + + @NotNull + @Override + protected LineageSearchService getLineageService() { + return lineageService; + } + + @NotNull + @Override + protected SearchService getSearchService() { + return searchService; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(lineageService); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java new file mode 100644 index 00000000000000..613ec5a26ff66e --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java @@ -0,0 +1,66 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.search.LineageServiceTestBase; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + + +@Import({ElasticSearchSuite.class, SearchCommonTestConfiguration.class, SearchTestContainerConfiguration.class}) +public class LineageServiceElasticSearchTest extends LineageServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + @Autowired + private SearchConfiguration _searchConfiguration; + @Autowired + private CustomSearchConfiguration _customSearchConfiguration; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @NotNull + @Override + protected SearchConfiguration getSearchConfiguration() { + return _searchConfiguration; + } + + @NotNull + @Override + protected CustomSearchConfiguration getCustomSearchConfiguration() { + return _customSearchConfiguration; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java new file mode 100644 index 00000000000000..855f46d239118b --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java @@ -0,0 +1,45 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase; +import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; + +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import lombok.Getter; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + + +/** + * Runs sample data fixture tests for Elasticsearch test container + */ +@Getter +@Import({ElasticSearchSuite.class, SampleDataFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class SampleDataFixtureElasticSearchTest extends SampleDataFixtureTestBase { + @Autowired + private RestHighLevelClient searchClient; + + @Autowired + @Qualifier("sampleDataSearchService") + protected SearchService searchService; + + @Autowired + @Qualifier("sampleDataEntityClient") + protected EntityClient entityClient; + + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry entityRegistry; + + @Test + public void initTest() { + assertNotNull(searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java new file mode 100644 index 00000000000000..1a6a20cd9df9de --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java @@ -0,0 +1,35 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.search.query.SearchDAOTestBase; +import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; +import com.linkedin.metadata.utils.elasticsearch.IndexConvention; + +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import lombok.Getter; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + + +@Getter +@Import({ElasticSearchSuite.class, SampleDataFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class SearchDAOElasticSearchTest extends SearchDAOTestBase { + @Autowired + private RestHighLevelClient searchClient; + @Autowired + private SearchConfiguration searchConfiguration; + @Autowired + @Qualifier("sampleDataIndexConvention") + IndexConvention indexConvention; + + @Test + public void initTest() { + assertNotNull(searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java new file mode 100644 index 00000000000000..a9e9feac280072 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java @@ -0,0 +1,65 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.search.SearchServiceTestBase; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({ElasticSearchSuite.class, SearchCommonTestConfiguration.class, SearchTestContainerConfiguration.class}) +public class SearchServiceElasticSearchTest extends SearchServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + @Autowired + private SearchConfiguration _searchConfiguration; + @Autowired + private CustomSearchConfiguration _customSearchConfiguration; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @NotNull + @Override + protected SearchConfiguration getSearchConfiguration() { + return _searchConfiguration; + } + + @NotNull + @Override + protected CustomSearchConfiguration getCustomSearchConfiguration() { + return _customSearchConfiguration; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java new file mode 100644 index 00000000000000..7365887fb9b2e3 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java @@ -0,0 +1,47 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.systemmetadata.SystemMetadataServiceTestBase; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + + +@Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) +public class SystemMetadataServiceElasticSearchTest extends SystemMetadataServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java new file mode 100644 index 00000000000000..bec610b20dca1e --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java @@ -0,0 +1,65 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.TestEntityTestBase; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({ElasticSearchSuite.class, SearchCommonTestConfiguration.class, SearchTestContainerConfiguration.class}) +public class TestEntityElasticSearchTest extends TestEntityTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + @Autowired + private SearchConfiguration _searchConfiguration; + @Autowired + private CustomSearchConfiguration _customSearchConfiguration; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @NotNull + @Override + protected SearchConfiguration getSearchConfiguration() { + return _searchConfiguration; + } + + @NotNull + @Override + protected CustomSearchConfiguration getCustomSearchConfiguration() { + return _customSearchConfiguration; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java new file mode 100644 index 00000000000000..5b85904edc9232 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java @@ -0,0 +1,46 @@ +package com.linkedin.metadata.search.elasticsearch; + +import com.linkedin.metadata.timeseries.search.TimeseriesAspectServiceTestBase; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) +public class TimeseriesAspectServiceElasticSearchTest extends TimeseriesAspectServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java deleted file mode 100644 index b506051e9bb5d8..00000000000000 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java +++ /dev/null @@ -1,312 +0,0 @@ -package com.linkedin.metadata.search.elasticsearch.query; - -import com.linkedin.metadata.config.search.SearchConfiguration; -import com.datahub.test.Snapshot; -import com.google.common.collect.ImmutableList; -import com.linkedin.data.template.LongMap; -import com.linkedin.data.template.StringArray; -import com.linkedin.metadata.ESSampleDataFixture; -import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; -import com.linkedin.metadata.query.filter.Condition; -import com.linkedin.metadata.query.filter.ConjunctiveCriterion; -import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; -import com.linkedin.metadata.query.filter.CriterionArray; -import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.search.AggregationMetadata; -import com.linkedin.metadata.search.AggregationMetadataArray; -import com.linkedin.metadata.search.FilterValueArray; -import com.linkedin.metadata.search.SearchEntityArray; -import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.search.SearchResultMetadata; -import com.linkedin.metadata.utils.SearchUtil; -import com.linkedin.metadata.utils.elasticsearch.IndexConvention; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Import; -import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; -import org.testng.annotations.Test; - -import com.linkedin.metadata.query.filter.Criterion; -import org.springframework.beans.factory.annotation.Qualifier; - -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.utils.SearchUtil.*; -import static org.testng.Assert.*; - - -@Import(ESSampleDataFixture.class) -public class ESSearchDAOTest extends AbstractTestNGSpringContextTests { - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private SearchConfiguration _searchConfiguration; - @Autowired - @Qualifier("sampleDataIndexConvention") - IndexConvention _indexConvention; - EntityRegistry _entityRegistry = new SnapshotEntityRegistry(new Snapshot()); - - - - @Test - public void testTransformFilterForEntitiesNoChange() { - Criterion c = new Criterion().setValue("urn:li:tag:abc").setValues( - new StringArray(ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("tags.keyword"); - - Filter f = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(c)))); - - Filter transformedFilter = SearchUtil.transformFilterForEntities(f, _indexConvention); - assertEquals(f, transformedFilter); - } - - @Test - public void testTransformFilterForEntitiesNullFilter() { - Filter transformedFilter = SearchUtil.transformFilterForEntities(null, _indexConvention); - assertNotNull(_indexConvention); - assertEquals(null, transformedFilter); - } - - @Test - public void testTransformFilterForEntitiesWithChanges() { - - Criterion c = new Criterion().setValue("dataset").setValues( - new StringArray(ImmutableList.of("dataset")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("_entityType"); - - Filter f = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(c)))); - Filter originalF = null; - try { - originalF = f.copy(); - } catch (CloneNotSupportedException e) { - fail(e.getMessage()); - } - assertEquals(f, originalF); - - Filter transformedFilter = SearchUtil.transformFilterForEntities(f, _indexConvention); - assertNotEquals(originalF, transformedFilter); - - Criterion expectedNewCriterion = new Criterion().setValue("smpldat_datasetindex_v2").setValues( - new StringArray(ImmutableList.of("smpldat_datasetindex_v2")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("_index"); - - Filter expectedNewFilter = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(expectedNewCriterion)))); - - assertEquals(expectedNewFilter, transformedFilter); - } - - @Test - public void testTransformFilterForEntitiesWithUnderscore() { - - Criterion c = new Criterion().setValue("data_job").setValues( - new StringArray(ImmutableList.of("data_job")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("_entityType"); - - Filter f = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(c)))); - Filter originalF = null; - try { - originalF = f.copy(); - } catch (CloneNotSupportedException e) { - fail(e.getMessage()); - } - assertEquals(f, originalF); - - Filter transformedFilter = SearchUtil.transformFilterForEntities(f, _indexConvention); - assertNotEquals(originalF, transformedFilter); - - Criterion expectedNewCriterion = new Criterion().setValue("smpldat_datajobindex_v2").setValues( - new StringArray(ImmutableList.of("smpldat_datajobindex_v2")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("_index"); - - Filter expectedNewFilter = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(expectedNewCriterion)))); - - assertEquals(transformedFilter, expectedNewFilter); - } - - @Test - public void testTransformFilterForEntitiesWithSomeChanges() { - - Criterion criterionChanged = new Criterion().setValue("dataset").setValues( - new StringArray(ImmutableList.of("dataset")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("_entityType"); - Criterion criterionUnchanged = new Criterion().setValue("urn:li:tag:abc").setValues( - new StringArray(ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("tags.keyword"); - - Filter f = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(criterionChanged, criterionUnchanged)))); - Filter originalF = null; - try { - originalF = f.copy(); - } catch (CloneNotSupportedException e) { - fail(e.getMessage()); - } - assertEquals(f, originalF); - - Filter transformedFilter = SearchUtil.transformFilterForEntities(f, _indexConvention); - assertNotEquals(originalF, transformedFilter); - - Criterion expectedNewCriterion = new Criterion().setValue("smpldat_datasetindex_v2").setValues( - new StringArray(ImmutableList.of("smpldat_datasetindex_v2")) - ).setNegated(false).setCondition(Condition.EQUAL).setField("_index"); - - Filter expectedNewFilter = new Filter().setOr( - new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(expectedNewCriterion, criterionUnchanged)))); - - assertEquals(expectedNewFilter, transformedFilter); - } - - @Test - public void testTransformIndexIntoEntityNameSingle() { - ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, _searchClient, _indexConvention, false, - ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, null); - // Empty aggregations - final SearchResultMetadata searchResultMetadata = - new SearchResultMetadata().setAggregations(new AggregationMetadataArray()); - SearchResult result = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) - .setMetadata(searchResultMetadata) - .setFrom(0) - .setPageSize(100) - .setNumEntities(30); - SearchResult expectedResult = null; - try { - expectedResult = result.copy(); - } catch (CloneNotSupportedException e) { - fail(e.getMessage()); - } - assertEquals(expectedResult, searchDAO.transformIndexIntoEntityName(result)); - - // one facet, do not transform - Map aggMap = Map.of("urn:li:corpuser:datahub", Long.valueOf(3)); - - List aggregationMetadataList = new ArrayList<>(); - aggregationMetadataList.add(new AggregationMetadata().setName("owners") - .setDisplayName("Owned by") - .setAggregations(new LongMap(aggMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(aggMap, Collections.emptySet()))) - ); - searchResultMetadata.setAggregations(new AggregationMetadataArray(aggregationMetadataList)); - result.setMetadata(searchResultMetadata); - - try { - expectedResult = result.copy(); - } catch (CloneNotSupportedException e) { - fail(e.getMessage()); - } - assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); - - // one facet, transform - Map entityTypeMap = Map.of("smpldat_datasetindex_v2", Long.valueOf(3)); - - aggregationMetadataList = List.of(new AggregationMetadata().setName("_entityType") - .setDisplayName("Type") - .setAggregations(new LongMap(entityTypeMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(entityTypeMap, Collections.emptySet()))) - ); - searchResultMetadata.setAggregations(new AggregationMetadataArray(aggregationMetadataList)); - result.setMetadata(searchResultMetadata); - - Map expectedEntityTypeMap = Map.of("dataset", Long.valueOf(3)); - - List expectedAggregationMetadataList = List.of( - new AggregationMetadata().setName("_entityType") - .setDisplayName("Type") - .setAggregations(new LongMap(expectedEntityTypeMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(expectedEntityTypeMap, Collections.emptySet()))) - ); - expectedResult.setMetadata(new SearchResultMetadata().setAggregations(new AggregationMetadataArray(expectedAggregationMetadataList))); - assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); - } - - @Test - public void testTransformIndexIntoEntityNameNested() { - ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, _searchClient, _indexConvention, false, - ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, null); - // One nested facet - Map entityTypeMap = Map.of( - String.format("smpldat_datasetindex_v2%surn:li:corpuser:datahub", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), - String.format("smpldat_datasetindex_v2%surn:li:corpuser:bfoo", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), - "smpldat_datasetindex_v2", Long.valueOf(20) - ); - List aggregationMetadataList = List.of(new AggregationMetadata().setName("_entityType␞owners") - .setDisplayName("Type␞Owned By") - .setAggregations(new LongMap(entityTypeMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(entityTypeMap, Collections.emptySet()))) - ); - SearchResult result = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) - .setMetadata(new SearchResultMetadata().setAggregations( - new AggregationMetadataArray(aggregationMetadataList) - )) - .setFrom(0) - .setPageSize(100) - .setNumEntities(50); - - Map expectedEntityTypeMap = Map.of( - String.format("dataset%surn:li:corpuser:datahub", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), - String.format("dataset%surn:li:corpuser:bfoo", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), - "dataset", Long.valueOf(20) - ); - - List expectedAggregationMetadataList = List.of(new AggregationMetadata().setName("_entityType␞owners") - .setDisplayName("Type␞Owned By") - .setAggregations(new LongMap(expectedEntityTypeMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(expectedEntityTypeMap, Collections.emptySet()))) - ); - SearchResult expectedResult = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) - .setMetadata(new SearchResultMetadata().setAggregations( - new AggregationMetadataArray(expectedAggregationMetadataList))) - .setFrom(0) - .setPageSize(100) - .setNumEntities(50); - assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); - - // One nested facet, opposite order - entityTypeMap = Map.of( - String.format("urn:li:corpuser:datahub%ssmpldat_datasetindex_v2", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), - String.format("urn:li:corpuser:datahub%ssmpldat_chartindex_v2", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), - "urn:li:corpuser:datahub", Long.valueOf(20) - ); - aggregationMetadataList = List.of(new AggregationMetadata().setName("owners␞_entityType") - .setDisplayName("Owned By␞Type") - .setAggregations(new LongMap(entityTypeMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(entityTypeMap, Collections.emptySet()))) - ); - result = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) - .setMetadata(new SearchResultMetadata().setAggregations( - new AggregationMetadataArray(aggregationMetadataList) - )) - .setFrom(0) - .setPageSize(100) - .setNumEntities(50); - - expectedEntityTypeMap = Map.of( - String.format("urn:li:corpuser:datahub%sdataset", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), - String.format("urn:li:corpuser:datahub%schart", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), - "urn:li:corpuser:datahub", Long.valueOf(20) - ); - - expectedAggregationMetadataList = List.of(new AggregationMetadata().setName("owners␞_entityType") - .setDisplayName("Owned By␞Type") - .setAggregations(new LongMap(expectedEntityTypeMap)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(expectedEntityTypeMap, Collections.emptySet()))) - ); - expectedResult = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) - .setMetadata(new SearchResultMetadata().setAggregations( - new AggregationMetadataArray(expectedAggregationMetadataList))) - .setFrom(0) - .setPageSize(100) - .setNumEntities(50); - assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); - } - - -} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/ElasticSearchGoldenTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java similarity index 74% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/ElasticSearchGoldenTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java index d720c95fef84d0..ed81f3cebd027a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/ElasticSearchGoldenTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java @@ -1,60 +1,47 @@ -package com.linkedin.metadata.search.elasticsearch.fixtures; +package com.linkedin.metadata.search.fixtures; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.ESSampleDataFixture; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.MatchedFieldArray; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; +import javax.annotation.Nonnull; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.linkedin.metadata.ESTestUtils.*; +import static io.datahubproject.test.search.SearchTestUtils.searchAcrossCustomEntities; +import static io.datahubproject.test.search.SearchTestUtils.searchAcrossEntities; import static org.testng.Assert.assertTrue; -import static org.testng.AssertJUnit.*; +import static org.testng.AssertJUnit.assertNotNull; -@Import(ESSampleDataFixture.class) -public class ElasticSearchGoldenTest extends AbstractTestNGSpringContextTests { +abstract public class GoldenTestBase extends AbstractTestNGSpringContextTests { private static final List SEARCHABLE_LONGTAIL_ENTITIES = Stream.of(EntityType.CHART, EntityType.CONTAINER, EntityType.DASHBOARD, EntityType.DATASET, EntityType.DOMAIN, EntityType.TAG ).map(EntityTypeMapper::getName) .collect(Collectors.toList()); - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - @Qualifier("longTailSearchService") - protected SearchService searchService; + @Nonnull + abstract protected EntityRegistry getEntityRegistry(); - @Autowired - @Qualifier("longTailEntityClient") - protected EntityClient entityClient; - - @Autowired - @Qualifier("longTailEntityRegistry") - private EntityRegistry entityRegistry; + @Nonnull + abstract protected SearchService getSearchService(); @Test public void testNameMatchPetProfiles() { /* Searching for "pet profiles" should return "pet_profiles" as the first 2 search results */ - assertNotNull(searchService); - assertNotNull(entityRegistry); - SearchResult searchResult = searchAcrossCustomEntities(searchService, "pet profiles", SEARCHABLE_LONGTAIL_ENTITIES); + assertNotNull(getSearchService()); + assertNotNull(getEntityRegistry()); + SearchResult searchResult = searchAcrossCustomEntities(getSearchService(), "pet profiles", SEARCHABLE_LONGTAIL_ENTITIES); assertTrue(searchResult.getEntities().size() >= 2); Urn firstResultUrn = searchResult.getEntities().get(0).getEntity(); Urn secondResultUrn = searchResult.getEntities().get(1).getEntity(); @@ -68,8 +55,8 @@ public void testNameMatchPetProfile() { /* Searching for "pet profile" should return "pet_profiles" as the first 2 search results */ - assertNotNull(searchService); - SearchResult searchResult = searchAcrossEntities(searchService, "pet profile", SEARCHABLE_LONGTAIL_ENTITIES); + assertNotNull(getSearchService()); + SearchResult searchResult = searchAcrossEntities(getSearchService(), "pet profile", SEARCHABLE_LONGTAIL_ENTITIES); assertTrue(searchResult.getEntities().size() >= 2); Urn firstResultUrn = searchResult.getEntities().get(0).getEntity(); Urn secondResultUrn = searchResult.getEntities().get(1).getEntity(); @@ -84,8 +71,8 @@ public void testGlossaryTerms() { Searching for "ReturnRate" should return all tables that have the glossary term applied before anything else */ - assertNotNull(searchService); - SearchResult searchResult = searchAcrossEntities(searchService, "ReturnRate", SEARCHABLE_LONGTAIL_ENTITIES); + assertNotNull(getSearchService()); + SearchResult searchResult = searchAcrossEntities(getSearchService(), "ReturnRate", SEARCHABLE_LONGTAIL_ENTITIES); SearchEntityArray entities = searchResult.getEntities(); assertTrue(searchResult.getEntities().size() >= 4); MatchedFieldArray firstResultMatchedFields = entities.get(0).getMatchedFields(); @@ -105,8 +92,8 @@ public void testNameMatchPartiallyQualified() { Searching for "analytics.pet_details" (partially qualified) should return the fully qualified table name as the first search results before any others */ - assertNotNull(searchService); - SearchResult searchResult = searchAcrossEntities(searchService, "analytics.pet_details", SEARCHABLE_LONGTAIL_ENTITIES); + assertNotNull(getSearchService()); + SearchResult searchResult = searchAcrossEntities(getSearchService(), "analytics.pet_details", SEARCHABLE_LONGTAIL_ENTITIES); assertTrue(searchResult.getEntities().size() >= 2); Urn firstResultUrn = searchResult.getEntities().get(0).getEntity(); Urn secondResultUrn = searchResult.getEntities().get(1).getEntity(); @@ -121,8 +108,8 @@ public void testNameMatchCollaborativeActionitems() { Searching for "collaborative actionitems" should return "collaborative_actionitems" as the first search result, followed by "collaborative_actionitems_old" */ - assertNotNull(searchService); - SearchResult searchResult = searchAcrossEntities(searchService, "collaborative actionitems", SEARCHABLE_LONGTAIL_ENTITIES); + assertNotNull(getSearchService()); + SearchResult searchResult = searchAcrossEntities(getSearchService(), "collaborative actionitems", SEARCHABLE_LONGTAIL_ENTITIES); assertTrue(searchResult.getEntities().size() >= 2); Urn firstResultUrn = searchResult.getEntities().get(0).getEntity(); Urn secondResultUrn = searchResult.getEntities().get(1).getEntity(); @@ -144,13 +131,17 @@ public void testNameMatchCustomerOrders() { Searching for "customer orders" should return "customer_orders" as the first search result, not suffixed by anything */ - assertNotNull(searchService); - SearchResult searchResult = searchAcrossEntities(searchService, "customer orders", SEARCHABLE_LONGTAIL_ENTITIES); + assertNotNull(getSearchService()); + SearchResult searchResult = searchAcrossEntities(getSearchService(), "customer orders", SEARCHABLE_LONGTAIL_ENTITIES); assertTrue(searchResult.getEntities().size() >= 2); Urn firstResultUrn = searchResult.getEntities().get(0).getEntity(); // Checks that the table name is not suffixed with anything - assertTrue(firstResultUrn.toString().contains("customer_orders,")); + assertTrue(firstResultUrn.toString().contains("customer_orders,"), + "Expected firstResultUrn to contain `customer_orders,` but results are " + + searchResult.getEntities().stream() + .map(e -> String.format("(Score: %s Urn: %s)", e.getScore(), e.getEntity().getId())) + .collect(Collectors.joining(", "))); Double firstResultScore = searchResult.getEntities().get(0).getScore(); Double secondResultScore = searchResult.getEntities().get(1).getScore(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/LineageDataFixtureTestBase.java similarity index 52% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/LineageDataFixtureTestBase.java index 55f7d4618f4794..eaf8feedeb6ed5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/LineageDataFixtureTestBase.java @@ -1,50 +1,43 @@ -package com.linkedin.metadata.search.elasticsearch.fixtures; +package com.linkedin.metadata.search.fixtures; import com.linkedin.common.urn.Urn; -import com.linkedin.metadata.ESSearchLineageFixture; -import com.linkedin.metadata.ESTestUtils; import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; +import javax.annotation.Nonnull; import java.net.URISyntaxException; -import static com.linkedin.metadata.ESTestUtils.lineage; +import static io.datahubproject.test.search.SearchTestUtils.lineage; +import static io.datahubproject.test.search.SearchTestUtils.searchAcrossEntities; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +abstract public class LineageDataFixtureTestBase extends AbstractTestNGSpringContextTests { -@Import(ESSearchLineageFixture.class) -public class SearchLineageDataFixtureTests extends AbstractTestNGSpringContextTests { + @Nonnull + abstract protected LineageSearchService getLineageService(); - @Autowired - @Qualifier("searchLineageSearchService") - protected SearchService searchService; - - @Autowired - @Qualifier("searchLineageLineageSearchService") - protected LineageSearchService lineageService; + @Nonnull + abstract protected SearchService getSearchService(); @Test public void testFixtureInitialization() { - assertNotNull(searchService); - SearchResult noResult = ESTestUtils.searchAcrossEntities(searchService, "no results"); + assertNotNull(getSearchService()); + SearchResult noResult = searchAcrossEntities(getSearchService(), "no results"); assertEquals(noResult.getEntities().size(), 0); - SearchResult result = ESTestUtils.searchAcrossEntities(searchService, "e3859789eed1cef55288b44f016ee08290d9fd08973e565c112d8"); + SearchResult result = searchAcrossEntities(getSearchService(), "e3859789eed1cef55288b44f016ee08290d9fd08973e565c112d8"); assertEquals(result.getEntities().size(), 1); assertEquals(result.getEntities().get(0).getEntity().toString(), "urn:li:dataset:(urn:li:dataPlatform:9cf8c96,e3859789eed1cef55288b44f016ee08290d9fd08973e565c112d8,PROD)"); - LineageSearchResult lineageResult = lineage(lineageService, result.getEntities().get(0).getEntity(), 1); + LineageSearchResult lineageResult = lineage(getLineageService(), result.getEntities().get(0).getEntity(), 1); assertEquals(lineageResult.getEntities().size(), 10); } @@ -54,15 +47,15 @@ public void testDatasetLineage() throws URISyntaxException { "urn:li:dataset:(urn:li:dataPlatform:9cf8c96,e3859789eed1cef55288b44f016ee08290d9fd08973e565c112d8,PROD)"); // 1 hops - LineageSearchResult lineageResult = lineage(lineageService, testUrn, 1); + LineageSearchResult lineageResult = lineage(getLineageService(), testUrn, 1); assertEquals(lineageResult.getEntities().size(), 10); // 2 hops - lineageResult = lineage(lineageService, testUrn, 2); + lineageResult = lineage(getLineageService(), testUrn, 2); assertEquals(lineageResult.getEntities().size(), 5); // 3 hops - lineageResult = lineage(lineageService, testUrn, 3); + lineageResult = lineage(getLineageService(), testUrn, 3); assertEquals(lineageResult.getEntities().size(), 12); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java similarity index 81% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index 450378b247cea9..16605048102965 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.search.elasticsearch.fixtures; +package com.linkedin.metadata.search.fixtures; import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; @@ -12,7 +12,6 @@ import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; import com.linkedin.datahub.graphql.types.dataset.DatasetType; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.ESSampleDataFixture; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -28,22 +27,19 @@ import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; - import com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig; import com.linkedin.r2.RemoteInvocationException; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.AnalyzeRequest; -import org.elasticsearch.client.indices.AnalyzeResponse; -import org.elasticsearch.client.indices.GetMappingsRequest; -import org.elasticsearch.client.indices.GetMappingsResponse; import org.junit.Assert; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Import; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.AnalyzeRequest; +import org.opensearch.client.indices.AnalyzeResponse; +import org.opensearch.client.indices.GetMappingsRequest; +import org.opensearch.client.indices.GetMappingsResponse; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -54,36 +50,36 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestUtils.*; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchQueryBuilder.STRUCTURED_QUERY_PREFIX; -import static com.linkedin.metadata.utils.SearchUtil.*; +import static com.linkedin.metadata.utils.SearchUtil.AGGREGATION_SEPARATOR_CHAR; +import static io.datahubproject.test.search.SearchTestUtils.autocomplete; +import static io.datahubproject.test.search.SearchTestUtils.scroll; +import static io.datahubproject.test.search.SearchTestUtils.search; +import static io.datahubproject.test.search.SearchTestUtils.searchAcrossEntities; +import static io.datahubproject.test.search.SearchTestUtils.searchStructured; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.assertFalse; - -@Import(ESSampleDataFixture.class) -public class SampleDataFixtureTests extends AbstractTestNGSpringContextTests { - private static final Authentication AUTHENTICATION = +abstract public class SampleDataFixtureTestBase extends AbstractTestNGSpringContextTests { + protected static final Authentication AUTHENTICATION = new Authentication(new Actor(ActorType.USER, "test"), ""); - @Autowired - private RestHighLevelClient _searchClient; + @Nonnull + abstract protected EntityRegistry getEntityRegistry(); - @Autowired - @Qualifier("sampleDataSearchService") - protected SearchService searchService; + @Nonnull + abstract protected SearchService getSearchService(); - @Autowired - @Qualifier("sampleDataEntityClient") - protected EntityClient entityClient; + @Nonnull + abstract protected EntityClient getEntityClient(); - @Autowired - @Qualifier("entityRegistry") - private EntityRegistry entityRegistry; + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); @Test public void testSearchFieldConfig() throws IOException { @@ -91,29 +87,29 @@ public void testSearchFieldConfig() throws IOException { For every field in every entity fixture, ensure proper detection of field types and analyzers */ Map fixtureEntities = new HashMap<>(); - fixtureEntities.put(entityRegistry.getEntitySpec("dataset"), "smpldat_datasetindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("chart"), "smpldat_chartindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("container"), "smpldat_containerindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("corpgroup"), "smpldat_corpgroupindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("corpuser"), "smpldat_corpuserindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("dashboard"), "smpldat_dashboardindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("dataflow"), "smpldat_dataflowindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("datajob"), "smpldat_datajobindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("domain"), "smpldat_domainindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("glossarynode"), "smpldat_glossarynodeindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("glossaryterm"), "smpldat_glossarytermindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("mlfeature"), "smpldat_mlfeatureindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("mlfeaturetable"), "smpldat_mlfeaturetableindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("mlmodelgroup"), "smpldat_mlmodelgroupindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("mlmodel"), "smpldat_mlmodelindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("mlprimarykey"), "smpldat_mlprimarykeyindex_v2"); - fixtureEntities.put(entityRegistry.getEntitySpec("tag"), "smpldat_tagindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("dataset"), "smpldat_datasetindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("chart"), "smpldat_chartindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("container"), "smpldat_containerindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("corpgroup"), "smpldat_corpgroupindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("corpuser"), "smpldat_corpuserindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("dashboard"), "smpldat_dashboardindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("dataflow"), "smpldat_dataflowindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("datajob"), "smpldat_datajobindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("domain"), "smpldat_domainindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("glossarynode"), "smpldat_glossarynodeindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("glossaryterm"), "smpldat_glossarytermindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("mlfeature"), "smpldat_mlfeatureindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("mlfeaturetable"), "smpldat_mlfeaturetableindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("mlmodelgroup"), "smpldat_mlmodelgroupindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("mlmodel"), "smpldat_mlmodelindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("mlprimarykey"), "smpldat_mlprimarykeyindex_v2"); + fixtureEntities.put(getEntityRegistry().getEntitySpec("tag"), "smpldat_tagindex_v2"); for (Map.Entry entry : fixtureEntities.entrySet()) { EntitySpec entitySpec = entry.getKey(); GetMappingsRequest req = new GetMappingsRequest().indices(entry.getValue()); - GetMappingsResponse resp = _searchClient.indices().getMapping(req, RequestOptions.DEFAULT); + GetMappingsResponse resp = getSearchClient().indices().getMapping(req, RequestOptions.DEFAULT); Map> mappings = (Map>) resp.mappings() .get(entry.getValue()).sourceAsMap().get("properties"); @@ -182,7 +178,7 @@ public void testSearchFieldConfig() throws IOException { public void testDatasetHasTags() throws IOException { GetMappingsRequest req = new GetMappingsRequest() .indices("smpldat_datasetindex_v2"); - GetMappingsResponse resp = _searchClient.indices().getMapping(req, RequestOptions.DEFAULT); + GetMappingsResponse resp = getSearchClient().indices().getMapping(req, RequestOptions.DEFAULT); Map> mappings = (Map>) resp.mappings() .get("smpldat_datasetindex_v2").sourceAsMap().get("properties"); assertTrue(mappings.containsKey("hasTags")); @@ -191,11 +187,11 @@ public void testDatasetHasTags() throws IOException { @Test public void testFixtureInitialization() { - assertNotNull(searchService); - SearchResult noResult = searchAcrossEntities(searchService, "no results"); + assertNotNull(getSearchService()); + SearchResult noResult = searchAcrossEntities(getSearchService(), "no results"); assertEquals(0, noResult.getEntities().size()); - final SearchResult result = searchAcrossEntities(searchService, "test"); + final SearchResult result = searchAcrossEntities(getSearchService(), "test"); Map expectedTypes = Map.of( "dataset", 13, @@ -209,7 +205,7 @@ public void testFixtureInitialization() { Map> actualTypes = new HashMap<>(); for (String key : expectedTypes.keySet()) { actualTypes.put(key, result.getEntities().stream() - .map(SearchEntity::getEntity).filter(entity -> key.equals(entity.getEntityType())).collect(Collectors.toList())); + .map(SearchEntity::getEntity).filter(entity -> key.equals(entity.getEntityType())).collect(Collectors.toList())); } expectedTypes.forEach((key, value) -> @@ -241,7 +237,7 @@ public void testDataPlatform() { .build(); expected.forEach((key, value) -> { - SearchResult result = searchAcrossEntities(searchService, key); + SearchResult result = searchAcrossEntities(getSearchService(), key); assertEquals(result.getEntities().size(), value.intValue(), String.format("Unexpected data platform `%s` hits.", key)); // max is 100 without pagination }); @@ -257,14 +253,14 @@ public void testUrn() { "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)" ).forEach(query -> - assertTrue(searchAcrossEntities(searchService, query).getEntities().size() >= 1, - String.format("Unexpected >1 urn result for `%s`", query)) + assertTrue(searchAcrossEntities(getSearchService(), query).getEntities().size() >= 1, + String.format("Unexpected >1 urn result for `%s`", query)) ); } @Test public void testExactTable() { - SearchResult results = searchAcrossEntities(searchService, "stg_customers"); + SearchResult results = searchAcrossEntities(getSearchService(), "stg_customers"); assertEquals(results.getEntities().size(), 1, "Unexpected single urn result for `stg_customers`"); assertEquals(results.getEntities().get(0).getEntity().toString(), "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)"); @@ -281,7 +277,7 @@ public void testStemming() { testSets.forEach(testSet -> { Integer expectedResults = null; for (String testQuery : testSet) { - SearchResult results = searchAcrossEntities(searchService, testQuery); + SearchResult results = searchAcrossEntities(getSearchService(), testQuery); assertTrue(results.hasEntities() && !results.getEntities().isEmpty(), String.format("Expected search results for `%s`", testQuery)); @@ -299,7 +295,7 @@ public void testStemmingOverride() throws IOException { Set testSet = Set.of("customer", "customers"); Set results = testSet.stream() - .map(test -> searchAcrossEntities(searchService, test)) + .map(test -> searchAcrossEntities(getSearchService(), test)) .collect(Collectors.toSet()); results.forEach(r -> assertTrue(r.hasEntities() && !r.getEntities().isEmpty(), "Expected search results")); @@ -352,7 +348,7 @@ public void testDelimitedSynonym() throws IOException { "customer acquisition cost" ); List resultCounts = testSet.stream().map(q -> { - SearchResult result = searchAcrossEntities(searchService, q); + SearchResult result = searchAcrossEntities(getSearchService(), q); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), "Expected search results for: " + q); return result.getEntities().size(); @@ -363,26 +359,26 @@ public void testDelimitedSynonym() throws IOException { public void testNegateAnalysis() throws IOException { String queryWithMinus = "logging_events -bckp"; AnalyzeRequest request = AnalyzeRequest.withIndexAnalyzer( - "smpldat_datasetindex_v2", - "query_word_delimited", queryWithMinus + "smpldat_datasetindex_v2", + "query_word_delimited", queryWithMinus ); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), - List.of("logging_events -bckp", "logging_ev", "-bckp", "log", "event", "bckp")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), + List.of("logging_events -bckp", "logging_ev", "-bckp", "log", "event", "bckp")); request = AnalyzeRequest.withIndexAnalyzer( - "smpldat_datasetindex_v2", - "word_gram_3", queryWithMinus + "smpldat_datasetindex_v2", + "word_gram_3", queryWithMinus ); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("logging events -bckp")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("logging events -bckp")); request = AnalyzeRequest.withIndexAnalyzer( - "smpldat_datasetindex_v2", - "word_gram_4", queryWithMinus + "smpldat_datasetindex_v2", + "word_gram_4", queryWithMinus ); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of()); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of()); } @@ -391,49 +387,49 @@ public void testWordGram() throws IOException { String text = "hello.cat_cool_customer"; AnalyzeRequest request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_2", text); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hello cat", "cat cool", "cool customer")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hello cat", "cat cool", "cool customer")); request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_3", text); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hello cat cool", "cat cool customer")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hello cat cool", "cat cool customer")); request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_4", text); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hello cat cool customer")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hello cat cool customer")); String testMoreSeparators = "quick.brown:fox jumped-LAZY_Dog"; request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_2", testMoreSeparators); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), - List.of("quick brown", "brown fox", "fox jumped", "jumped lazy", "lazy dog")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), + List.of("quick brown", "brown fox", "fox jumped", "jumped lazy", "lazy dog")); request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_3", testMoreSeparators); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), - List.of("quick brown fox", "brown fox jumped", "fox jumped lazy", "jumped lazy dog")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), + List.of("quick brown fox", "brown fox jumped", "fox jumped lazy", "jumped lazy dog")); request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_4", testMoreSeparators); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), - List.of("quick brown fox jumped", "brown fox jumped lazy", "fox jumped lazy dog")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), + List.of("quick brown fox jumped", "brown fox jumped lazy", "fox jumped lazy dog")); String textWithQuotesAndDuplicateWord = "\"my_db.my_exact_table\""; request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_2", textWithQuotesAndDuplicateWord); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("my db", "db my", "my exact", "exact table")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("my db", "db my", "my exact", "exact table")); request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_3", textWithQuotesAndDuplicateWord); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("my db my", "db my exact", "my exact table")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("my db my", "db my exact", "my exact table")); request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_4", textWithQuotesAndDuplicateWord); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("my db my exact", "db my exact table")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("my db my exact", "db my exact table")); String textWithParens = "(hi) there"; request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", "word_gram_2", textWithParens); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hi there")); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of("hi there")); String oneWordText = "hello"; for (String analyzer : List.of("word_gram_2", "word_gram_3", "word_gram_4")) { request = AnalyzeRequest.withIndexAnalyzer("smpldat_datasetindex_v2", analyzer, oneWordText); assertEquals(getTokens(request) - .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of()); + .map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()), List.of()); } } @@ -463,7 +459,7 @@ public void testUrnSynonym() throws IOException { "big query" ); List results = testSet.stream().map(query -> { - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), "Expected search results for: " + query); return result; }).collect(Collectors.toList()); @@ -504,9 +500,9 @@ public void testTokenizationWithNumber() throws IOException { ); List tokens = getTokens(request).map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList()); assertEquals(tokens, List.of( - "harshal-playground-306419", "harshal", "playground", "306419", - "test_schema", "test", "schema", - "austin311_deriv", "austin311", "deriv"), + "harshal-playground-306419", "harshal", "playground", "306419", + "test_schema", "test", "schema", + "austin311_deriv", "austin311", "deriv"), String.format("Unexpected tokens. Found %s", tokens)); request = AnalyzeRequest.withIndexAnalyzer( @@ -622,7 +618,7 @@ public void testChartAutoComplete() throws InterruptedException, IOException { List.of("B", "Ba", "Baz", "Baz ", "Baz C", "Baz Ch", "Baz Cha", "Baz Char", "Baz Chart", "Baz Chart ") .forEach(query -> { try { - AutoCompleteResults result = autocomplete(new ChartType(entityClient), query); + AutoCompleteResults result = autocomplete(new ChartType(getEntityClient()), query); assertTrue(result.getEntities().size() == 2, String.format("Expected 2 results for `%s` found %s", query, result.getEntities().size())); } catch (Exception e) { @@ -637,7 +633,7 @@ public void testDatasetAutoComplete() { "excess_deaths_de", "excess_deaths_der", "excess_deaths_derived") .forEach(query -> { try { - AutoCompleteResults result = autocomplete(new DatasetType(entityClient), query); + AutoCompleteResults result = autocomplete(new DatasetType(getEntityClient()), query); assertTrue(result.getEntities().size() >= 1, String.format("Expected >= 1 results for `%s` found %s", query, result.getEntities().size())); } catch (Exception e) { @@ -652,7 +648,7 @@ public void testContainerAutoComplete() { "container-autocomp-test") .forEach(query -> { try { - AutoCompleteResults result = autocomplete(new ContainerType(entityClient), query); + AutoCompleteResults result = autocomplete(new ContainerType(getEntityClient()), query); assertTrue(result.getEntities().size() >= 1, String.format("Expected >= 1 results for `%s` found %s", query, result.getEntities().size())); } catch (Exception e) { @@ -666,7 +662,7 @@ public void testGroupAutoComplete() { List.of("T", "Te", "Tes", "Test ", "Test G", "Test Gro", "Test Group ") .forEach(query -> { try { - AutoCompleteResults result = autocomplete(new CorpGroupType(entityClient), query); + AutoCompleteResults result = autocomplete(new CorpGroupType(getEntityClient()), query); assertTrue(result.getEntities().size() == 1, String.format("Expected 1 results for `%s` found %s", query, result.getEntities().size())); } catch (Exception e) { @@ -680,7 +676,7 @@ public void testUserAutoComplete() { List.of("D", "Da", "Dat", "Data ", "Data H", "Data Hu", "Data Hub", "Data Hub ") .forEach(query -> { try { - AutoCompleteResults result = autocomplete(new CorpUserType(entityClient, null), query); + AutoCompleteResults result = autocomplete(new CorpUserType(getEntityClient(), null), query); assertTrue(result.getEntities().size() >= 1, String.format("Expected at least 1 results for `%s` found %s", query, result.getEntities().size())); } catch (Exception e) { @@ -702,7 +698,7 @@ public void testSmokeTestQueries() { ); Map results = expectedFulltextMinimums.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchAcrossEntities(searchService, entry.getKey()))); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchAcrossEntities(getSearchService(), entry.getKey()))); results.forEach((key, value) -> { Integer actualCount = value.getEntities().size(); @@ -719,7 +715,7 @@ public void testSmokeTestQueries() { ); results = expectedStructuredMinimums.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchStructured(searchService, entry.getKey()))); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchStructured(getSearchService(), entry.getKey()))); results.forEach((key, value) -> { Integer actualCount = value.getEntities().size(); @@ -772,7 +768,7 @@ public void testUnderscore() throws IOException { @Test public void testFacets() { Set expectedFacets = Set.of("entity", "typeNames", "platform", "origin", "tags"); - SearchResult testResult = searchAcrossEntities(searchService, "cypress"); + SearchResult testResult = searchAcrossEntities(getSearchService(), "cypress"); expectedFacets.forEach(facet -> { assertTrue(testResult.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), String.format("Failed to find facet `%s` in %s", facet, @@ -780,7 +776,7 @@ public void testFacets() { .map(AggregationMetadata::getName).collect(Collectors.toList()))); }); AggregationMetadata entityAggMeta = testResult.getMetadata().getAggregations().stream().filter( - aggMeta -> aggMeta.getName().equals("entity")).findFirst().get(); + aggMeta -> aggMeta.getName().equals("entity")).findFirst().get(); Map expectedEntityTypeCounts = new HashMap<>(); expectedEntityTypeCounts.put("container", 0L); expectedEntityTypeCounts.put("corpuser", 0L); @@ -805,28 +801,28 @@ public void testFacets() { @Test public void testNestedAggregation() { Set expectedFacets = Set.of("platform"); - SearchResult testResult = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResult = searchAcrossEntities(getSearchService(), "cypress", List.copyOf(expectedFacets)); assertEquals(testResult.getMetadata().getAggregations().size(), 1); expectedFacets.forEach(facet -> { assertTrue(testResult.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), - String.format("Failed to find facet `%s` in %s", facet, - testResult.getMetadata().getAggregations().stream() - .map(AggregationMetadata::getName).collect(Collectors.toList()))); + String.format("Failed to find facet `%s` in %s", facet, + testResult.getMetadata().getAggregations().stream() + .map(AggregationMetadata::getName).collect(Collectors.toList()))); }); expectedFacets = Set.of("platform", "typeNames", "_entityType", "entity"); - SearchResult testResult2 = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResult2 = searchAcrossEntities(getSearchService(), "cypress", List.copyOf(expectedFacets)); assertEquals(testResult2.getMetadata().getAggregations().size(), 4); expectedFacets.forEach(facet -> { assertTrue(testResult2.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), - String.format("Failed to find facet `%s` in %s", facet, - testResult2.getMetadata().getAggregations().stream() - .map(AggregationMetadata::getName).collect(Collectors.toList()))); + String.format("Failed to find facet `%s` in %s", facet, + testResult2.getMetadata().getAggregations().stream() + .map(AggregationMetadata::getName).collect(Collectors.toList()))); }); AggregationMetadata entityTypeAggMeta = testResult2.getMetadata().getAggregations().stream().filter( - aggMeta -> aggMeta.getName().equals("_entityType")).findFirst().get(); + aggMeta -> aggMeta.getName().equals("_entityType")).findFirst().get(); AggregationMetadata entityAggMeta = testResult2.getMetadata().getAggregations().stream().filter( - aggMeta -> aggMeta.getName().equals("entity")).findFirst().get(); + aggMeta -> aggMeta.getName().equals("entity")).findFirst().get(); assertEquals(entityTypeAggMeta.getAggregations(), entityAggMeta.getAggregations()); Map expectedEntityTypeCounts = new HashMap<>(); expectedEntityTypeCounts.put("container", 0L); @@ -849,24 +845,24 @@ public void testNestedAggregation() { assertEquals(entityTypeAggMeta.getAggregations(), expectedEntityTypeCounts); expectedFacets = Set.of("platform", "typeNames", "entity"); - SearchResult testResult3 = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResult3 = searchAcrossEntities(getSearchService(), "cypress", List.copyOf(expectedFacets)); assertEquals(testResult3.getMetadata().getAggregations().size(), 4); expectedFacets.forEach(facet -> { assertTrue(testResult3.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), - String.format("Failed to find facet `%s` in %s", facet, - testResult3.getMetadata().getAggregations().stream() - .map(AggregationMetadata::getName).collect(Collectors.toList()))); + String.format("Failed to find facet `%s` in %s", facet, + testResult3.getMetadata().getAggregations().stream() + .map(AggregationMetadata::getName).collect(Collectors.toList()))); }); AggregationMetadata entityTypeAggMeta3 = testResult3.getMetadata().getAggregations().stream().filter( - aggMeta -> aggMeta.getName().equals("_entityType")).findFirst().get(); + aggMeta -> aggMeta.getName().equals("_entityType")).findFirst().get(); AggregationMetadata entityAggMeta3 = testResult3.getMetadata().getAggregations().stream().filter( - aggMeta -> aggMeta.getName().equals("entity")).findFirst().get(); + aggMeta -> aggMeta.getName().equals("entity")).findFirst().get(); assertEquals(entityTypeAggMeta3.getAggregations(), entityAggMeta3.getAggregations()); assertEquals(entityTypeAggMeta3.getAggregations(), expectedEntityTypeCounts); String singleNestedFacet = String.format("_entityType%sowners", AGGREGATION_SEPARATOR_CHAR); expectedFacets = Set.of(singleNestedFacet); - SearchResult testResultSingleNested = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResultSingleNested = searchAcrossEntities(getSearchService(), "cypress", List.copyOf(expectedFacets)); assertEquals(testResultSingleNested.getMetadata().getAggregations().size(), 1); Map expectedNestedFacetCounts = new HashMap<>(); expectedNestedFacetCounts.put("datajob␞urn:li:corpuser:datahub", 2L); @@ -885,17 +881,17 @@ public void testNestedAggregation() { assertEquals(testResultSingleNested.getMetadata().getAggregations().get(0).getAggregations(), expectedNestedFacetCounts); expectedFacets = Set.of("platform", singleNestedFacet, "typeNames", "origin"); - SearchResult testResultNested = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResultNested = searchAcrossEntities(getSearchService(), "cypress", List.copyOf(expectedFacets)); assertEquals(testResultNested.getMetadata().getAggregations().size(), 4); expectedFacets.forEach(facet -> { assertTrue(testResultNested.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), - String.format("Failed to find facet `%s` in %s", facet, - testResultNested.getMetadata().getAggregations().stream() - .map(AggregationMetadata::getName).collect(Collectors.toList()))); + String.format("Failed to find facet `%s` in %s", facet, + testResultNested.getMetadata().getAggregations().stream() + .map(AggregationMetadata::getName).collect(Collectors.toList()))); }); List expectedNestedAgg = testResultNested.getMetadata().getAggregations().stream().filter( - agg -> agg.getName().equals(singleNestedFacet)).collect(Collectors.toList()); + agg -> agg.getName().equals(singleNestedFacet)).collect(Collectors.toList()); assertEquals(expectedNestedAgg.size(), 1); AggregationMetadata nestedAgg = expectedNestedAgg.get(0); assertEquals(nestedAgg.getDisplayName(), String.format("Type%sOwned By", AGGREGATION_SEPARATOR_CHAR)); @@ -959,7 +955,7 @@ public void testScrollAcrossEntities() throws IOException { int totalResults = 0; String scrollId = null; do { - ScrollResult result = scroll(searchService, query, batchSize, scrollId); + ScrollResult result = scroll(getSearchService(), query, batchSize, scrollId); int numResults = result.hasEntities() ? result.getEntities().size() : 0; assertTrue(numResults <= batchSize); totalResults += numResults; @@ -972,13 +968,13 @@ public void testScrollAcrossEntities() throws IOException { @Test public void testSearchAcrossMultipleEntities() { String query = "logging_events"; - SearchResult result = search(searchService, query); + SearchResult result = search(getSearchService(), query); assertEquals((int) result.getNumEntities(), 8); - result = search(searchService, List.of(DATASET_ENTITY_NAME, DATA_JOB_ENTITY_NAME), query); + result = search(getSearchService(), List.of(DATASET_ENTITY_NAME, DATA_JOB_ENTITY_NAME), query); assertEquals((int) result.getNumEntities(), 8); - result = search(searchService, List.of(DATASET_ENTITY_NAME), query); + result = search(getSearchService(), List.of(DATASET_ENTITY_NAME), query); assertEquals((int) result.getNumEntities(), 4); - result = search(searchService, List.of(DATA_JOB_ENTITY_NAME), query); + result = search(getSearchService(), List.of(DATA_JOB_ENTITY_NAME), query); assertEquals((int) result.getNumEntities(), 4); } @@ -1046,7 +1042,7 @@ public void testFragmentUrns() { ); testSet.forEach(query -> { - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected partial urn search results", query)); @@ -1064,7 +1060,7 @@ public void testPlatformTest() { List results = testFields.stream() .map(fieldName -> { final String query = String.format("%s:%s", fieldName, testPlatform.replaceAll(":", "\\\\:")); - SearchResult result = searchStructured(searchService, query); + SearchResult result = searchStructured(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1095,7 +1091,7 @@ public void testPlatformTest() { // Test field variations with/without .keyword List entityClientResults = testFilters.stream().map(filter -> { try { - return entityClient.search("dataset", "*", filter, null, 0, 100, + return getEntityClient().search("dataset", "*", filter, null, 0, 100, AUTHENTICATION, new SearchFlags().setFulltext(fulltextFlag)); } catch (RemoteInvocationException e) { throw new RuntimeException(e); @@ -1112,7 +1108,7 @@ public void testPlatformTest() { @Test public void testStructQueryFieldMatch() { String query = STRUCTURED_QUERY_PREFIX + "name: customers"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1125,7 +1121,7 @@ public void testStructQueryFieldMatch() { @Test public void testStructQueryFieldPrefixMatch() { String query = STRUCTURED_QUERY_PREFIX + "name: customers*"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1138,7 +1134,7 @@ public void testStructQueryFieldPrefixMatch() { @Test public void testStructQueryCustomPropertiesKeyPrefix() { String query = STRUCTURED_QUERY_PREFIX + "customProperties: node_type=*"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1151,7 +1147,7 @@ public void testStructQueryCustomPropertiesKeyPrefix() { @Test public void testStructQueryCustomPropertiesMatch() { String query = STRUCTURED_QUERY_PREFIX + "customProperties: node_type=model"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1169,7 +1165,7 @@ public void testCustomPropertiesQuoted() { ); Map results = expectedResults.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchAcrossEntities(searchService, entry.getKey()))); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchAcrossEntities(getSearchService(), entry.getKey()))); results.forEach((key, value) -> { Integer actualCount = value.getEntities().size(); @@ -1183,7 +1179,7 @@ public void testCustomPropertiesQuoted() { @Test public void testStructQueryFieldPaths() { String query = STRUCTURED_QUERY_PREFIX + "fieldPaths: customer_id"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1196,7 +1192,7 @@ public void testStructQueryFieldPaths() { @Test public void testStructQueryBoolean() { String query = STRUCTURED_QUERY_PREFIX + "editedFieldTags:urn\\:li\\:tag\\:Legacy OR tags:urn\\:li\\:tag\\:testTag"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1206,7 +1202,7 @@ public void testStructQueryBoolean() { assertEquals(result.getEntities().size(), 2); query = STRUCTURED_QUERY_PREFIX + "editedFieldTags:urn\\:li\\:tag\\:Legacy"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1216,7 +1212,7 @@ public void testStructQueryBoolean() { assertEquals(result.getEntities().size(), 1); query = STRUCTURED_QUERY_PREFIX + "tags:urn\\:li\\:tag\\:testTag"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1229,7 +1225,7 @@ public void testStructQueryBoolean() { @Test public void testStructQueryBrowsePaths() { String query = STRUCTURED_QUERY_PREFIX + "browsePaths:*/dbt/*"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1242,7 +1238,7 @@ public void testStructQueryBrowsePaths() { @Test public void testOr() { String query = "stg_customers | logging_events"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1250,7 +1246,7 @@ public void testOr() { assertEquals(result.getEntities().size(), 9); query = "stg_customers"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1258,7 +1254,7 @@ public void testOr() { assertEquals(result.getEntities().size(), 1); query = "logging_events"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1269,7 +1265,7 @@ public void testOr() { @Test public void testNegate() { String query = "logging_events -bckp"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1277,7 +1273,7 @@ public void testNegate() { assertEquals(result.getEntities().size(), 7); query = "logging_events"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1288,7 +1284,7 @@ public void testNegate() { @Test public void testPrefix() { String query = "bigquery"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1296,7 +1292,7 @@ public void testPrefix() { assertEquals(result.getEntities().size(), 8); query = "big*"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1307,7 +1303,7 @@ public void testPrefix() { @Test public void testParens() { String query = "dbt | (bigquery + covid19)"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1315,7 +1311,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 11); query = "dbt"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1323,7 +1319,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 9); query = "bigquery + covid19"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1331,7 +1327,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 2); query = "bigquery"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1339,7 +1335,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 8); query = "covid19"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1349,55 +1345,55 @@ public void testParens() { @Test public void testGram() { String query = "jaffle shop customers"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", - "Expected exact match in 1st position"); + "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "Expected exact match in 1st position"); query = "shop customers source"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", - "Expected ngram match in 1st position"); + "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", + "Expected ngram match in 1st position"); query = "jaffle shop stg customers"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", - "Expected ngram match in 1st position"); + "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "Expected ngram match in 1st position"); query = "jaffle shop transformers customers"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", - "Expected ngram match in 1st position"); + "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "Expected ngram match in 1st position"); query = "shop raw customers"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", - "Expected ngram match in 1st position"); + "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "Expected ngram match in 1st position"); } @Test public void testPrefixVsExact() { String query = "\"customers\""; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1415,7 +1411,7 @@ public void testPrefixVsExact() { public void testPrefixVsExactCaseSensitivity() { List insensitiveExactMatches = List.of("testExactMatchCase", "testexactmatchcase", "TESTEXACTMATCHCASE"); for (String query : insensitiveExactMatches) { - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1432,33 +1428,33 @@ public void testPrefixVsExactCaseSensitivity() { @Test public void testColumnExactMatch() { String query = "unit_data"; - SearchResult result = searchAcrossEntities(searchService, query); + SearchResult result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), - String.format("%s - Expected search results to include matched fields", query)); + String.format("%s - Expected search results to include matched fields", query)); assertTrue(result.getEntities().size() > 2, - String.format("%s - Expected search results to have at least two results", query)); + String.format("%s - Expected search results to have at least two results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:testOnly," + query + ",PROD)", - "Expected table name exact match first"); + "urn:li:dataset:(urn:li:dataPlatform:testOnly," + query + ",PROD)", + "Expected table name exact match first"); query = "special_column_only_present_here_info"; - result = searchAcrossEntities(searchService, query); + result = searchAcrossEntities(getSearchService(), query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), - String.format("%s - Expected search results", query)); + String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), - String.format("%s - Expected search results to include matched fields", query)); + String.format("%s - Expected search results to include matched fields", query)); assertTrue(result.getEntities().size() > 2, - String.format("%s - Expected search results to have at least two results", query)); + String.format("%s - Expected search results to have at least two results", query)); assertEquals(result.getEntities().get(0).getEntity().toString(), - "urn:li:dataset:(urn:li:dataPlatform:testOnly," + "important_units" + ",PROD)", - "Expected table with column name exact match first"); + "urn:li:dataset:(urn:li:dataPlatform:testOnly," + "important_units" + ",PROD)", + "Expected table with column name exact match first"); } private Stream getTokens(AnalyzeRequest request) throws IOException { - return _searchClient.indices().analyze(request, RequestOptions.DEFAULT).getTokens().stream(); + return getSearchClient().indices().analyze(request, RequestOptions.DEFAULT).getTokens().stream(); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java similarity index 85% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java index 2416280cb8f931..4472af339c074d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java @@ -1,43 +1,40 @@ -package com.linkedin.metadata.search.elasticsearch.indexbuilder; +package com.linkedin.metadata.search.indexbuilder; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.google.common.collect.ImmutableMap; -import com.linkedin.metadata.ESTestConfiguration; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.systemmetadata.SystemMetadataMappingsBuilder; import com.linkedin.metadata.version.GitVersion; -import java.util.Optional; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.client.IndicesClient; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.indices.GetIndexResponse; -import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.rest.RestStatus; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Import; +import org.opensearch.OpenSearchException; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.client.IndicesClient; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.client.indices.GetIndexResponse; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.rest.RestStatus; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.*; + +abstract public class IndexBuilderTestBase extends AbstractTestNGSpringContextTests { -@Import(ESTestConfiguration.class) -public class ESIndexBuilderTest extends AbstractTestNGSpringContextTests { + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); - @Autowired - private RestHighLevelClient _searchClient; private static IndicesClient _indexClient; private static final String TEST_INDEX_NAME = "esindex_builder_test"; private static ESIndexBuilder testDefaultBuilder; @@ -45,9 +42,9 @@ public class ESIndexBuilderTest extends AbstractTestNGSpringContextTests { @BeforeClass public void setup() { - _indexClient = _searchClient.indices(); + _indexClient = getSearchClient().indices(); GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); - testDefaultBuilder = new ESIndexBuilder(_searchClient, 1, 0, 0, + testDefaultBuilder = new ESIndexBuilder(getSearchClient(), 1, 0, 0, 0, Map.of(), false, false, new ElasticSearchConfiguration(), gitVersion); } @@ -65,7 +62,7 @@ public static void wipe() throws Exception { }); _indexClient.delete(new DeleteIndexRequest(TEST_INDEX_NAME), RequestOptions.DEFAULT); - } catch (ElasticsearchException exception) { + } catch (OpenSearchException exception) { if (exception.status() != RestStatus.NOT_FOUND) { throw exception; } @@ -79,7 +76,7 @@ public static GetIndexResponse getTestIndex() throws IOException { @Test public void testESIndexBuilderCreation() throws Exception { GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); - ESIndexBuilder customIndexBuilder = new ESIndexBuilder(_searchClient, 2, 0, 1, + ESIndexBuilder customIndexBuilder = new ESIndexBuilder(getSearchClient(), 2, 0, 1, 0, Map.of(), false, false, new ElasticSearchConfiguration(), gitVersion); customIndexBuilder.buildIndex(TEST_INDEX_NAME, Map.of(), Map.of()); @@ -93,7 +90,7 @@ public void testESIndexBuilderCreation() throws Exception { @Test public void testMappingReindex() throws Exception { GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); - ESIndexBuilder enabledMappingReindex = new ESIndexBuilder(_searchClient, 1, 0, 0, + ESIndexBuilder enabledMappingReindex = new ESIndexBuilder(getSearchClient(), 1, 0, 0, 0, Map.of(), false, true, new ElasticSearchConfiguration(), gitVersion); @@ -111,7 +108,7 @@ public void testMappingReindex() throws Exception { Map newProps = ((Map) SystemMetadataMappingsBuilder.getMappings().get("properties")) .entrySet().stream() .map(m -> !m.getKey().equals("urn") ? m - : Map.entry("urn", ImmutableMap.builder().put("type", "wildcard").build())) + : Map.entry("urn", ImmutableMap.builder().put("type", "text").build())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); enabledMappingReindex.buildIndex(TEST_INDEX_NAME, Map.of("properties", newProps), Map.of()); @@ -134,7 +131,7 @@ public void testSettingsNumberOfShardsReindex() throws Exception { String expectedShards = "5"; GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); - ESIndexBuilder changedShardBuilder = new ESIndexBuilder(_searchClient, + ESIndexBuilder changedShardBuilder = new ESIndexBuilder(getSearchClient(), Integer.parseInt(expectedShards), testDefaultBuilder.getNumReplicas(), testDefaultBuilder.getNumRetries(), @@ -162,7 +159,7 @@ public void testSettingsNumberOfShardsReindex() throws Exception { public void testSettingsNoReindex() throws Exception { GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); List noReindexBuilders = List.of( - new ESIndexBuilder(_searchClient, + new ESIndexBuilder(getSearchClient(), testDefaultBuilder.getNumShards(), testDefaultBuilder.getNumReplicas() + 1, testDefaultBuilder.getNumRetries(), @@ -170,7 +167,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), true, false, new ElasticSearchConfiguration(), gitVersion), - new ESIndexBuilder(_searchClient, + new ESIndexBuilder(getSearchClient(), testDefaultBuilder.getNumShards(), testDefaultBuilder.getNumReplicas(), testDefaultBuilder.getNumRetries(), @@ -178,7 +175,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), true, false, new ElasticSearchConfiguration(), gitVersion), - new ESIndexBuilder(_searchClient, + new ESIndexBuilder(getSearchClient(), testDefaultBuilder.getNumShards() + 1, testDefaultBuilder.getNumReplicas(), testDefaultBuilder.getNumRetries(), @@ -186,7 +183,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), false, false, new ElasticSearchConfiguration(), gitVersion), - new ESIndexBuilder(_searchClient, + new ESIndexBuilder(getSearchClient(), testDefaultBuilder.getNumShards(), testDefaultBuilder.getNumReplicas() + 1, testDefaultBuilder.getNumRetries(), diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java similarity index 98% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java index 0b331855492990..0d2ce236d9f54d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java @@ -1,8 +1,10 @@ -package com.linkedin.metadata.search.elasticsearch.indexbuilder; +package com.linkedin.metadata.search.indexbuilder; import com.google.common.collect.ImmutableMap; import com.linkedin.metadata.TestEntitySpecBuilder; import java.util.Map; + +import com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java new file mode 100644 index 00000000000000..3896ba749e85e0 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java @@ -0,0 +1,44 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.fixtures.GoldenTestBase; +import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + +@Import({OpenSearchSuite.class, SampleDataFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class GoldenOpenSearchTest extends GoldenTestBase { + + @Autowired + @Qualifier("longTailSearchService") + protected SearchService searchService; + + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry entityRegistry; + + + @NotNull + @Override + protected EntityRegistry getEntityRegistry() { + return entityRegistry; + } + + @NotNull + @Override + protected SearchService getSearchService() { + return searchService; + } + + @Test + public void initTest() { + assertNotNull(searchService); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java new file mode 100644 index 00000000000000..312b56364bd914 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java @@ -0,0 +1,30 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.search.indexbuilder.IndexBuilderTestBase; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + + +@Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) +public class IndexBuilderOpenSearchTest extends IndexBuilderTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @Test + public void initTest() { + assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java new file mode 100644 index 00000000000000..6fc0677ad6e39c --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java @@ -0,0 +1,43 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.search.LineageSearchService; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.fixtures.LineageDataFixtureTestBase; +import io.datahubproject.test.fixtures.search.SearchLineageFixtureConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + + +@Import({OpenSearchSuite.class, SearchLineageFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class LineageDataFixtureOpenSearchTest extends LineageDataFixtureTestBase { + + @Autowired + @Qualifier("searchLineageSearchService") + protected SearchService searchService; + + @Autowired + @Qualifier("searchLineageLineageSearchService") + protected LineageSearchService lineageService; + + @NotNull + @Override + protected LineageSearchService getLineageService() { + return lineageService; + } + + @NotNull + @Override + protected SearchService getSearchService() { + return searchService; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(lineageService); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java new file mode 100644 index 00000000000000..1a6242c2211fd5 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java @@ -0,0 +1,65 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.LineageServiceTestBase; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({OpenSearchSuite.class, SearchCommonTestConfiguration.class, SearchTestContainerConfiguration.class}) +public class LineageServiceOpenSearchTest extends LineageServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + @Autowired + private SearchConfiguration _searchConfiguration; + @Autowired + private CustomSearchConfiguration _customSearchConfiguration; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @NotNull + @Override + protected SearchConfiguration getSearchConfiguration() { + return _searchConfiguration; + } + + @NotNull + @Override + protected CustomSearchConfiguration getCustomSearchConfiguration() { + return _customSearchConfiguration; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/OpenSearchSuite.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/OpenSearchSuite.java new file mode 100644 index 00000000000000..559c623c97d5a6 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/OpenSearchSuite.java @@ -0,0 +1,31 @@ +package com.linkedin.metadata.search.opensearch; + +import io.datahubproject.test.search.OpenSearchTestContainer; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testcontainers.containers.GenericContainer; +import org.testng.annotations.AfterSuite; + +@TestConfiguration +public class OpenSearchSuite extends AbstractTestNGSpringContextTests { + + private static final OpenSearchTestContainer OPENSEARCH_TEST_CONTAINER; + private static GenericContainer container; + static { + OPENSEARCH_TEST_CONTAINER = new OpenSearchTestContainer(); + } + + @AfterSuite + public void after() { + OPENSEARCH_TEST_CONTAINER.stopContainer(); + } + + @Bean(name = "testSearchContainer") + public GenericContainer testSearchContainer() { + if (container == null) { + container = OPENSEARCH_TEST_CONTAINER.startContainer(); + } + return container; + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java new file mode 100644 index 00000000000000..081eb5f70fc855 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java @@ -0,0 +1,44 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase; +import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import lombok.Getter; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + + +/** + * Runs sample data fixture tests for Opensearch test container + */ +@Getter +@Import({OpenSearchSuite.class, SampleDataFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class SampleDataFixtureOpenSearchTest extends SampleDataFixtureTestBase { + @Autowired + private RestHighLevelClient searchClient; + + @Autowired + @Qualifier("sampleDataSearchService") + protected SearchService searchService; + + @Autowired + @Qualifier("sampleDataEntityClient") + protected EntityClient entityClient; + + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry entityRegistry; + + @Test + public void initTest() { + assertNotNull(searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java new file mode 100644 index 00000000000000..0b166975da0d1d --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java @@ -0,0 +1,33 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.search.query.SearchDAOTestBase; +import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import lombok.Getter; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Import; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + + +@Getter +@Import({OpenSearchSuite.class, SampleDataFixtureConfiguration.class, SearchTestContainerConfiguration.class}) +public class SearchDAOOpenSearchTest extends SearchDAOTestBase { + @Autowired + private RestHighLevelClient searchClient; + @Autowired + private SearchConfiguration searchConfiguration; + @Autowired + @Qualifier("sampleDataIndexConvention") + IndexConvention indexConvention; + + @Test + public void initTest() { + assertNotNull(searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java new file mode 100644 index 00000000000000..8a55ba7b37ef91 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java @@ -0,0 +1,65 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.SearchServiceTestBase; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({OpenSearchSuite.class, SearchCommonTestConfiguration.class, SearchTestContainerConfiguration.class}) +public class SearchServiceOpenSearchTest extends SearchServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + @Autowired + private SearchConfiguration _searchConfiguration; + @Autowired + private CustomSearchConfiguration _customSearchConfiguration; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @NotNull + @Override + protected SearchConfiguration getSearchConfiguration() { + return _searchConfiguration; + } + + @NotNull + @Override + protected CustomSearchConfiguration getCustomSearchConfiguration() { + return _customSearchConfiguration; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java new file mode 100644 index 00000000000000..f0bb8e1c124799 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java @@ -0,0 +1,47 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import com.linkedin.metadata.systemmetadata.SystemMetadataServiceTestBase; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + + +@Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) +public class SystemMetadataServiceOpenSearchTest extends SystemMetadataServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java new file mode 100644 index 00000000000000..467f7fb43be1b0 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java @@ -0,0 +1,65 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.TestEntityTestBase; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({OpenSearchSuite.class, SearchCommonTestConfiguration.class, SearchTestContainerConfiguration.class}) +public class TestEntityOpenSearchTest extends TestEntityTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + @Autowired + private SearchConfiguration _searchConfiguration; + @Autowired + private CustomSearchConfiguration _customSearchConfiguration; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @NotNull + @Override + protected SearchConfiguration getSearchConfiguration() { + return _searchConfiguration; + } + + @NotNull + @Override + protected CustomSearchConfiguration getCustomSearchConfiguration() { + return _customSearchConfiguration; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java new file mode 100644 index 00000000000000..3333b9f0942f54 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java @@ -0,0 +1,46 @@ +package com.linkedin.metadata.search.opensearch; + +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import com.linkedin.metadata.timeseries.search.TimeseriesAspectServiceTestBase; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +@Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) +public class TimeseriesAspectServiceOpenSearchTest extends TimeseriesAspectServiceTestBase { + + @Autowired + private RestHighLevelClient _searchClient; + @Autowired + private ESBulkProcessor _bulkProcessor; + @Autowired + private ESIndexBuilder _esIndexBuilder; + + @NotNull + @Override + protected RestHighLevelClient getSearchClient() { + return _searchClient; + } + + @NotNull + @Override + protected ESBulkProcessor getBulkProcessor() { + return _bulkProcessor; + } + + @NotNull + @Override + protected ESIndexBuilder getIndexBuilder() { + return _esIndexBuilder; + } + + @Test + public void initTest() { + AssertJUnit.assertNotNull(_searchClient); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java similarity index 86% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAOTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java index 0a5f71345751b4..91e7747afb4a1c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java @@ -1,7 +1,8 @@ -package com.linkedin.metadata.search.elasticsearch.query; +package com.linkedin.metadata.search.query; import com.linkedin.common.urn.Urn; -import com.linkedin.metadata.ESTestConfiguration; +import com.linkedin.metadata.search.elasticsearch.query.ESBrowseDAO; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.entity.TestEntityRegistry; @@ -11,11 +12,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; @@ -28,8 +29,8 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -@Import(ESTestConfiguration.class) -public class ESBrowseDAOTest extends AbstractTestNGSpringContextTests { +@Import(SearchCommonTestConfiguration.class) +public class BrowseDAOTest extends AbstractTestNGSpringContextTests { private RestHighLevelClient _mockClient; private ESBrowseDAO _browseDAO; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java new file mode 100644 index 00000000000000..2dbc142d45071a --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java @@ -0,0 +1,307 @@ +package com.linkedin.metadata.search.query; + +import com.datahub.test.Snapshot; +import com.google.common.collect.ImmutableList; +import com.linkedin.data.template.LongMap; +import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.AggregationMetadata; +import com.linkedin.metadata.search.AggregationMetadataArray; +import com.linkedin.metadata.search.FilterValueArray; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.SearchResultMetadata; +import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; +import com.linkedin.metadata.utils.SearchUtil; +import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static com.linkedin.metadata.Constants.ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH; +import static com.linkedin.metadata.utils.SearchUtil.AGGREGATION_SEPARATOR_CHAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; + +abstract public class SearchDAOTestBase extends AbstractTestNGSpringContextTests { + + abstract protected RestHighLevelClient getSearchClient(); + + abstract protected SearchConfiguration getSearchConfiguration(); + + abstract protected IndexConvention getIndexConvention(); + + EntityRegistry _entityRegistry = new SnapshotEntityRegistry(new Snapshot()); + + + @Test + public void testTransformFilterForEntitiesNoChange() { + Criterion c = new Criterion().setValue("urn:li:tag:abc").setValues( + new StringArray(ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("tags.keyword"); + + Filter f = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(c)))); + + Filter transformedFilter = SearchUtil.transformFilterForEntities(f, getIndexConvention()); + assertEquals(f, transformedFilter); + } + + @Test + public void testTransformFilterForEntitiesNullFilter() { + Filter transformedFilter = SearchUtil.transformFilterForEntities(null, getIndexConvention()); + assertNotNull(getIndexConvention()); + assertEquals(null, transformedFilter); + } + + @Test + public void testTransformFilterForEntitiesWithChanges() { + + Criterion c = new Criterion().setValue("dataset").setValues( + new StringArray(ImmutableList.of("dataset")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("_entityType"); + + Filter f = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(c)))); + Filter originalF = null; + try { + originalF = f.copy(); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + assertEquals(f, originalF); + + Filter transformedFilter = SearchUtil.transformFilterForEntities(f, getIndexConvention()); + assertNotEquals(originalF, transformedFilter); + + Criterion expectedNewCriterion = new Criterion().setValue("smpldat_datasetindex_v2").setValues( + new StringArray(ImmutableList.of("smpldat_datasetindex_v2")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("_index"); + + Filter expectedNewFilter = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(expectedNewCriterion)))); + + assertEquals(expectedNewFilter, transformedFilter); + } + + @Test + public void testTransformFilterForEntitiesWithUnderscore() { + + Criterion c = new Criterion().setValue("data_job").setValues( + new StringArray(ImmutableList.of("data_job")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("_entityType"); + + Filter f = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(c)))); + Filter originalF = null; + try { + originalF = f.copy(); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + assertEquals(f, originalF); + + Filter transformedFilter = SearchUtil.transformFilterForEntities(f, getIndexConvention()); + assertNotEquals(originalF, transformedFilter); + + Criterion expectedNewCriterion = new Criterion().setValue("smpldat_datajobindex_v2").setValues( + new StringArray(ImmutableList.of("smpldat_datajobindex_v2")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("_index"); + + Filter expectedNewFilter = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(expectedNewCriterion)))); + + assertEquals(transformedFilter, expectedNewFilter); + } + + @Test + public void testTransformFilterForEntitiesWithSomeChanges() { + + Criterion criterionChanged = new Criterion().setValue("dataset").setValues( + new StringArray(ImmutableList.of("dataset")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("_entityType"); + Criterion criterionUnchanged = new Criterion().setValue("urn:li:tag:abc").setValues( + new StringArray(ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("tags.keyword"); + + Filter f = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(criterionChanged, criterionUnchanged)))); + Filter originalF = null; + try { + originalF = f.copy(); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + assertEquals(f, originalF); + + Filter transformedFilter = SearchUtil.transformFilterForEntities(f, getIndexConvention()); + assertNotEquals(originalF, transformedFilter); + + Criterion expectedNewCriterion = new Criterion().setValue("smpldat_datasetindex_v2").setValues( + new StringArray(ImmutableList.of("smpldat_datasetindex_v2")) + ).setNegated(false).setCondition(Condition.EQUAL).setField("_index"); + + Filter expectedNewFilter = new Filter().setOr( + new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(expectedNewCriterion, criterionUnchanged)))); + + assertEquals(expectedNewFilter, transformedFilter); + } + + @Test + public void testTransformIndexIntoEntityNameSingle() { + ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, getSearchClient(), getIndexConvention(), false, + ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, getSearchConfiguration(), null); + // Empty aggregations + final SearchResultMetadata searchResultMetadata = + new SearchResultMetadata().setAggregations(new AggregationMetadataArray()); + SearchResult result = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) + .setMetadata(searchResultMetadata) + .setFrom(0) + .setPageSize(100) + .setNumEntities(30); + SearchResult expectedResult = null; + try { + expectedResult = result.copy(); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + assertEquals(expectedResult, searchDAO.transformIndexIntoEntityName(result)); + + // one facet, do not transform + Map aggMap = Map.of("urn:li:corpuser:datahub", Long.valueOf(3)); + + List aggregationMetadataList = new ArrayList<>(); + aggregationMetadataList.add(new AggregationMetadata().setName("owners") + .setDisplayName("Owned by") + .setAggregations(new LongMap(aggMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(aggMap, Collections.emptySet()))) + ); + searchResultMetadata.setAggregations(new AggregationMetadataArray(aggregationMetadataList)); + result.setMetadata(searchResultMetadata); + + try { + expectedResult = result.copy(); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); + + // one facet, transform + Map entityTypeMap = Map.of("smpldat_datasetindex_v2", Long.valueOf(3)); + + aggregationMetadataList = List.of(new AggregationMetadata().setName("_entityType") + .setDisplayName("Type") + .setAggregations(new LongMap(entityTypeMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(entityTypeMap, Collections.emptySet()))) + ); + searchResultMetadata.setAggregations(new AggregationMetadataArray(aggregationMetadataList)); + result.setMetadata(searchResultMetadata); + + Map expectedEntityTypeMap = Map.of("dataset", Long.valueOf(3)); + + List expectedAggregationMetadataList = List.of( + new AggregationMetadata().setName("_entityType") + .setDisplayName("Type") + .setAggregations(new LongMap(expectedEntityTypeMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(expectedEntityTypeMap, Collections.emptySet()))) + ); + expectedResult.setMetadata(new SearchResultMetadata().setAggregations(new AggregationMetadataArray(expectedAggregationMetadataList))); + assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); + } + + @Test + public void testTransformIndexIntoEntityNameNested() { + ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, getSearchClient(), getIndexConvention(), false, + ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, getSearchConfiguration(), null); + // One nested facet + Map entityTypeMap = Map.of( + String.format("smpldat_datasetindex_v2%surn:li:corpuser:datahub", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), + String.format("smpldat_datasetindex_v2%surn:li:corpuser:bfoo", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), + "smpldat_datasetindex_v2", Long.valueOf(20) + ); + List aggregationMetadataList = List.of(new AggregationMetadata().setName("_entityType␞owners") + .setDisplayName("Type␞Owned By") + .setAggregations(new LongMap(entityTypeMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(entityTypeMap, Collections.emptySet()))) + ); + SearchResult result = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) + .setMetadata(new SearchResultMetadata().setAggregations( + new AggregationMetadataArray(aggregationMetadataList) + )) + .setFrom(0) + .setPageSize(100) + .setNumEntities(50); + + Map expectedEntityTypeMap = Map.of( + String.format("dataset%surn:li:corpuser:datahub", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), + String.format("dataset%surn:li:corpuser:bfoo", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), + "dataset", Long.valueOf(20) + ); + + List expectedAggregationMetadataList = List.of(new AggregationMetadata().setName("_entityType␞owners") + .setDisplayName("Type␞Owned By") + .setAggregations(new LongMap(expectedEntityTypeMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(expectedEntityTypeMap, Collections.emptySet()))) + ); + SearchResult expectedResult = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) + .setMetadata(new SearchResultMetadata().setAggregations( + new AggregationMetadataArray(expectedAggregationMetadataList))) + .setFrom(0) + .setPageSize(100) + .setNumEntities(50); + assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); + + // One nested facet, opposite order + entityTypeMap = Map.of( + String.format("urn:li:corpuser:datahub%ssmpldat_datasetindex_v2", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), + String.format("urn:li:corpuser:datahub%ssmpldat_chartindex_v2", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), + "urn:li:corpuser:datahub", Long.valueOf(20) + ); + aggregationMetadataList = List.of(new AggregationMetadata().setName("owners␞_entityType") + .setDisplayName("Owned By␞Type") + .setAggregations(new LongMap(entityTypeMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(entityTypeMap, Collections.emptySet()))) + ); + result = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) + .setMetadata(new SearchResultMetadata().setAggregations( + new AggregationMetadataArray(aggregationMetadataList) + )) + .setFrom(0) + .setPageSize(100) + .setNumEntities(50); + + expectedEntityTypeMap = Map.of( + String.format("urn:li:corpuser:datahub%sdataset", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(3), + String.format("urn:li:corpuser:datahub%schart", AGGREGATION_SEPARATOR_CHAR), Long.valueOf(7), + "urn:li:corpuser:datahub", Long.valueOf(20) + ); + + expectedAggregationMetadataList = List.of(new AggregationMetadata().setName("owners␞_entityType") + .setDisplayName("Owned By␞Type") + .setAggregations(new LongMap(expectedEntityTypeMap)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(expectedEntityTypeMap, Collections.emptySet()))) + ); + expectedResult = new SearchResult().setEntities(new SearchEntityArray(new ArrayList<>())) + .setMetadata(new SearchResultMetadata().setAggregations( + new AggregationMetadataArray(expectedAggregationMetadataList))) + .setFrom(0) + .setPageSize(100) + .setNumEntities(50); + assertEquals(searchDAO.transformIndexIntoEntityName(result), expectedResult); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java similarity index 94% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index 36c8bb8f9a6764..66e7b62741f4c5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.search.elasticsearch.query.request; +package com.linkedin.metadata.search.query.request; import com.google.common.collect.ImmutableSet; import com.linkedin.metadata.config.search.SearchConfiguration; @@ -9,7 +9,9 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.elasticsearch.search.aggregations.AggregationBuilder; + +import com.linkedin.metadata.search.elasticsearch.query.request.AggregationQueryBuilder; +import org.opensearch.search.aggregations.AggregationBuilder; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java similarity index 88% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandlerTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java index be91cb02889508..34b98f38254cd6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java @@ -1,15 +1,17 @@ -package com.linkedin.metadata.search.elasticsearch.query.request; +package com.linkedin.metadata.search.query.request; import com.linkedin.metadata.TestEntitySpecBuilder; import java.util.List; import java.util.Map; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.MultiMatchQueryBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; + +import com.linkedin.metadata.search.elasticsearch.query.request.AutocompleteRequestHandler; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java similarity index 93% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java index 3dad9c59c6b53f..6b6664ffdf30ed 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.search.elasticsearch.query.request; +package com.linkedin.metadata.search.query.request; import com.linkedin.metadata.config.search.CustomConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; @@ -7,12 +7,14 @@ import com.linkedin.metadata.config.search.custom.QueryConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import org.elasticsearch.common.lucene.search.function.CombineFunction; -import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import com.linkedin.metadata.search.elasticsearch.query.request.CustomizedQueryHandler; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchQueryBuilder; +import org.opensearch.common.lucene.search.function.CombineFunction; +import org.opensearch.common.lucene.search.function.FunctionScoreQuery; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.opensearch.index.query.functionscore.ScoreFunctionBuilders; import org.testng.annotations.Test; import java.io.IOException; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java similarity index 95% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 8e73b0ceeae8d1..9c0815efdc8b46 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -1,8 +1,10 @@ -package com.linkedin.metadata.search.elasticsearch.query.request; +package com.linkedin.metadata.search.query.request; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.schema.PathSpec; -import com.linkedin.metadata.ESTestConfiguration; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchQueryBuilder; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import com.linkedin.metadata.config.search.CustomConfiguration; import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; @@ -26,15 +28,15 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.util.Pair; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder; -import org.elasticsearch.index.query.MatchPhraseQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.index.query.SimpleQueryStringBuilder; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; @@ -50,7 +52,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -@Import(ESTestConfiguration.class) +@Import(SearchCommonTestConfiguration.class) public class SearchQueryBuilderTest extends AbstractTestNGSpringContextTests { @Autowired diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java similarity index 95% rename from metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java index db56e2d34881b6..90c6c523c588ff 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java @@ -1,11 +1,12 @@ -package com.linkedin.metadata.search.elasticsearch.query.request; +package com.linkedin.metadata.search.query.request; import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.google.common.collect.ImmutableList; import com.linkedin.data.template.StringArray; -import com.linkedin.metadata.ESTestConfiguration; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchRequestHandler; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import com.linkedin.metadata.TestEntitySpecBuilder; import com.linkedin.metadata.config.search.WordGramConfiguration; import java.util.ArrayList; @@ -28,17 +29,17 @@ import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.ExistsQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.MultiMatchQueryBuilder; -import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.ExistsQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; @@ -48,7 +49,7 @@ import static org.testng.Assert.*; -@Import(ESTestConfiguration.class) +@Import(SearchCommonTestConfiguration.class) public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests { @Autowired private EntityRegistry entityRegistry; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index 4f364c246818f0..ddd75a152c3330 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -4,7 +4,7 @@ import com.linkedin.data.template.StringArray; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.Criterion; -import org.elasticsearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java similarity index 84% rename from metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataServiceTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java index 6e116df5b2906b..e6a9bd7d198f7a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java @@ -1,6 +1,5 @@ package com.linkedin.metadata.systemmetadata; -import com.linkedin.metadata.ESTestConfiguration; import com.linkedin.metadata.run.AspectRowSummary; import com.linkedin.metadata.run.IngestionRunSummary; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -9,9 +8,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import com.linkedin.mxe.SystemMetadata; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Import; +import org.opensearch.client.RestHighLevelClient; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -20,18 +17,20 @@ import javax.annotation.Nonnull; import java.util.List; -import static com.linkedin.metadata.ESTestConfiguration.syncAfterWrite; +import static io.datahubproject.test.search.SearchTestUtils.syncAfterWrite; import static org.testng.Assert.assertEquals; -@Import(ESTestConfiguration.class) -public class ElasticSearchSystemMetadataServiceTest extends AbstractTestNGSpringContextTests { +abstract public class SystemMetadataServiceTestBase extends AbstractTestNGSpringContextTests { + + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); + + @Nonnull + abstract protected ESBulkProcessor getBulkProcessor(); + + @Nonnull + abstract protected ESIndexBuilder getIndexBuilder(); - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private ESBulkProcessor _bulkProcessor; - @Autowired - private ESIndexBuilder _esIndexBuilder; private final IndexConvention _indexConvention = new IndexConventionImpl("es_system_metadata_service_test"); private ElasticSearchSystemMetadataService _client; @@ -49,8 +48,8 @@ public void wipe() throws Exception { @Nonnull private ElasticSearchSystemMetadataService buildService() { - ESSystemMetadataDAO dao = new ESSystemMetadataDAO(_searchClient, _indexConvention, _bulkProcessor, 1); - return new ElasticSearchSystemMetadataService(_bulkProcessor, _indexConvention, dao, _esIndexBuilder); + ESSystemMetadataDAO dao = new ESSystemMetadataDAO(getSearchClient(), _indexConvention, getBulkProcessor(), 1); + return new ElasticSearchSystemMetadataService(getBulkProcessor(), _indexConvention, dao, getIndexBuilder()); } @Test @@ -70,7 +69,7 @@ public void testListRuns() throws Exception { _client.insert(metadata2, "urn:li:chart:2", "chartKey"); _client.insert(metadata2, "urn:li:chart:2", "Ownership"); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); List runs = _client.listRuns(0, 20, false); @@ -99,7 +98,7 @@ public void testOverwriteRuns() throws Exception { _client.insert(metadata2, "urn:li:chart:2", "chartKey"); _client.insert(metadata2, "urn:li:chart:2", "Ownership"); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); List runs = _client.listRuns(0, 20, false); @@ -128,7 +127,7 @@ public void testFindByRunId() throws Exception { _client.insert(metadata2, "urn:li:chart:2", "chartKey"); _client.insert(metadata2, "urn:li:chart:2", "Ownership"); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); List rows = _client.findByRunId("abc-456", false, 0, ESUtils.MAX_RESULT_SIZE); @@ -156,11 +155,11 @@ public void testDelete() throws Exception { _client.insert(metadata2, "urn:li:chart:2", "chartKey"); _client.insert(metadata2, "urn:li:chart:2", "Ownership"); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); _client.deleteUrn("urn:li:chart:1"); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); List rows = _client.findByRunId("abc-456", false, 0, ESUtils.MAX_RESULT_SIZE); @@ -172,7 +171,7 @@ public void testDelete() throws Exception { public void testInsertNullData() throws Exception { _client.insert(null, "urn:li:chart:1", "chartKey"); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); List runs = _client.listRuns(0, 20, false); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java similarity index 97% rename from metadata-io/src/test/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectServiceTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java index d65234bf89d49c..cc60ba8679e1f0 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.timeseries.elastic; +package com.linkedin.metadata.timeseries.search; import com.datahub.test.BatchType; import com.datahub.test.ComplexNestedRecord; @@ -16,7 +16,6 @@ import com.linkedin.data.template.StringArrayArray; import com.linkedin.data.template.StringMap; import com.linkedin.data.template.StringMapArray; -import com.linkedin.metadata.ESTestConfiguration; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DataSchemaFactory; @@ -32,6 +31,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.search.utils.QueryUtils; +import com.linkedin.metadata.timeseries.elastic.ElasticSearchTimeseriesAspectService; import com.linkedin.metadata.timeseries.elastic.indexbuilder.TimeseriesAspectIndexBuilders; import com.linkedin.metadata.timeseries.transformer.TimeseriesAspectTransformer; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -45,9 +45,7 @@ import com.linkedin.timeseries.GroupingBucket; import com.linkedin.timeseries.GroupingBucketType; import com.linkedin.timeseries.TimeWindowSize; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Import; +import org.opensearch.client.RestHighLevelClient; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -60,15 +58,15 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestConfiguration.syncAfterWrite; +import static com.linkedin.metadata.Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH; +import static com.linkedin.metadata.Constants.MAX_JACKSON_STRING_SIZE; +import static io.datahubproject.test.search.SearchTestUtils.syncAfterWrite; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -@Import(ESTestConfiguration.class) -public class ElasticSearchTimeseriesAspectServiceTest extends AbstractTestNGSpringContextTests { +abstract public class TimeseriesAspectServiceTestBase extends AbstractTestNGSpringContextTests { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); static { int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); @@ -85,12 +83,15 @@ public class ElasticSearchTimeseriesAspectServiceTest extends AbstractTestNGSpri private static final String ES_FIELD_TIMESTAMP = "timestampMillis"; private static final String ES_FIELD_STAT = "stat"; - @Autowired - private RestHighLevelClient _searchClient; - @Autowired - private ESBulkProcessor _bulkProcessor; - @Autowired - private ESIndexBuilder _esIndexBuilder; + @Nonnull + abstract protected RestHighLevelClient getSearchClient(); + + @Nonnull + abstract protected ESBulkProcessor getBulkProcessor(); + + @Nonnull + abstract protected ESIndexBuilder getIndexBuilder(); + private EntityRegistry _entityRegistry; private IndexConvention _indexConvention; private ElasticSearchTimeseriesAspectService _elasticSearchTimeseriesAspectService; @@ -116,9 +117,9 @@ public void setup() { @Nonnull private ElasticSearchTimeseriesAspectService buildService() { - return new ElasticSearchTimeseriesAspectService(_searchClient, _indexConvention, - new TimeseriesAspectIndexBuilders(_esIndexBuilder, _entityRegistry, - _indexConvention), _entityRegistry, _bulkProcessor, 1); + return new ElasticSearchTimeseriesAspectService(getSearchClient(), _indexConvention, + new TimeseriesAspectIndexBuilders(getIndexBuilder(), _entityRegistry, + _indexConvention), _entityRegistry, getBulkProcessor(), 1); } /* @@ -190,7 +191,7 @@ public void testUpsertProfiles() throws Exception { } }); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); } @Test(groups = "upsertUniqueMessageId") @@ -216,7 +217,7 @@ public void testUpsertProfilesWithUniqueMessageIds() throws Exception { } }); - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); List resultAspects = _elasticSearchTimeseriesAspectService.getAspectValues(urn, ENTITY_NAME, ASPECT_NAME, null, null, @@ -860,7 +861,7 @@ public void testCountByFilter() { @Test(groups = {"testCountAfterDelete"}, dependsOnGroups = {"deleteAspectValues1"}) public void testCountByFilterAfterDelete() throws InterruptedException { - syncAfterWrite(_bulkProcessor); + syncAfterWrite(getBulkProcessor()); // Test with filter Criterion hasUrnCriterion = new Criterion().setField("urn").setCondition(Condition.EQUAL).setValue(TEST_URN.toString()); diff --git a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/Utils.java b/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/Utils.java deleted file mode 100644 index f96a6c50af33da..00000000000000 --- a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/Utils.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.datahub.test.fixtures.elasticsearch; - -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - -import static com.linkedin.metadata.Constants.*; - - -public class Utils { - private Utils() { - - } - final public static String FIXTURE_BASE = "src/test/resources/elasticsearch"; - - final public static ObjectMapper OBJECT_MAPPER = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - static { - int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); - OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); - } -} diff --git a/metadata-io/src/test/java/io/datahub/test/DataGenerator.java b/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java similarity index 99% rename from metadata-io/src/test/java/io/datahub/test/DataGenerator.java rename to metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java index 3b374993cde16b..cfa9c1258583d7 100644 --- a/metadata-io/src/test/java/io/datahub/test/DataGenerator.java +++ b/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java @@ -1,4 +1,4 @@ -package io.datahub.test; +package io.datahubproject.test; import com.linkedin.common.AuditStamp; import com.linkedin.common.GlossaryTermAssociation; @@ -111,7 +111,8 @@ public Stream> generateMCPs(String entityName, long }).map(mcp -> { // Expand with default aspects per normal return Stream.concat(Stream.of(mcp), - AspectUtils.getAdditionalChanges(mcp, entityService, true).stream()).collect(Collectors.toList()); + AspectUtils.getAdditionalChanges(mcp, entityService, true).stream()) + .collect(Collectors.toList()); }); } diff --git a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/EntityExporter.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/EntityExporter.java similarity index 81% rename from metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/EntityExporter.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/EntityExporter.java index 5c34b9f549d9f5..18fbf86f8668de 100644 --- a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/EntityExporter.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/EntityExporter.java @@ -1,15 +1,15 @@ -package io.datahub.test.fixtures.elasticsearch; +package io.datahubproject.test.fixtures.search; import lombok.Builder; import lombok.NonNull; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetMappingsRequest; -import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetMappingsRequest; +import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; import java.io.IOException; import java.util.Set; diff --git a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/FixtureReader.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/FixtureReader.java similarity index 93% rename from metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/FixtureReader.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/FixtureReader.java index a0c551b28b507a..1b804a2346883f 100644 --- a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/FixtureReader.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/FixtureReader.java @@ -1,12 +1,12 @@ -package io.datahub.test.fixtures.elasticsearch; +package io.datahubproject.test.fixtures.search; import com.fasterxml.jackson.core.JsonProcessingException; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import lombok.Builder; import lombok.NonNull; import org.apache.commons.io.FilenameUtils; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.common.xcontent.XContentType; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.common.xcontent.XContentType; import java.io.BufferedInputStream; import java.io.BufferedReader; @@ -23,12 +23,12 @@ import java.util.stream.Stream; import java.util.zip.GZIPInputStream; -import static io.datahub.test.fixtures.elasticsearch.Utils.OBJECT_MAPPER; +import static io.datahubproject.test.fixtures.search.SearchFixtureUtils.OBJECT_MAPPER; @Builder public class FixtureReader { @Builder.Default - private String inputBase = Utils.FIXTURE_BASE; + private String inputBase = SearchFixtureUtils.FIXTURE_BASE; @NonNull private ESBulkProcessor bulkProcessor; @NonNull diff --git a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/FixtureWriter.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/FixtureWriter.java similarity index 75% rename from metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/FixtureWriter.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/FixtureWriter.java index 36b057bc22a370..0aefa006421fc4 100644 --- a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/FixtureWriter.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/FixtureWriter.java @@ -1,13 +1,13 @@ -package io.datahub.test.fixtures.elasticsearch; +package io.datahubproject.test.fixtures.search; import com.fasterxml.jackson.core.JsonProcessingException; import lombok.Builder; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; import javax.annotation.Nullable; import java.io.BufferedWriter; @@ -15,8 +15,6 @@ import java.io.IOException; import java.util.function.BiConsumer; -import static io.datahub.test.fixtures.elasticsearch.Utils.OBJECT_MAPPER; - /** * */ @@ -26,7 +24,7 @@ public class FixtureWriter { private RestHighLevelClient client; @Builder.Default - private String outputBase = Utils.FIXTURE_BASE; + private String outputBase = SearchFixtureUtils.FIXTURE_BASE; public void write(SearchRequest searchRequest, String relativeOutput, boolean append) { write(searchRequest, relativeOutput, append, null, null, null); @@ -53,14 +51,14 @@ public void write(SearchRequest searchRequest, String relativeOutput, boo if (outputType == null) { bw.write(hit.getSourceAsString()); } else { - O doc = OBJECT_MAPPER.readValue(hit.getSourceAsString(), outputType); - bw.write(OBJECT_MAPPER.writeValueAsString(doc)); + O doc = SearchFixtureUtils.OBJECT_MAPPER.readValue(hit.getSourceAsString(), outputType); + bw.write(SearchFixtureUtils.OBJECT_MAPPER.writeValueAsString(doc)); } bw.newLine(); // Fire callback if (callback != null) { - callback.accept(hit, OBJECT_MAPPER.readValue(hit.getSourceAsString(), callbackType)); + callback.accept(hit, SearchFixtureUtils.OBJECT_MAPPER.readValue(hit.getSourceAsString(), callbackType)); } } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/LineageExporter.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java similarity index 95% rename from metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/LineageExporter.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java index 3b236b36cdce18..5db07ee6fb8bca 100644 --- a/metadata-io/src/test/java/io/datahub/test/fixtures/elasticsearch/LineageExporter.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java @@ -1,14 +1,14 @@ -package io.datahub.test.fixtures.elasticsearch; +package io.datahubproject.test.fixtures.search; import com.google.common.collect.Lists; import lombok.Builder; import lombok.NonNull; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; import java.net.URLDecoder; import java.net.URLEncoder; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java similarity index 94% rename from metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java index ef9992db1fb25a..45bbd912bc7942 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java @@ -1,5 +1,6 @@ -package com.linkedin.metadata; +package io.datahubproject.test.fixtures.search; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.search.CustomConfiguration; @@ -30,9 +31,9 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import com.linkedin.metadata.version.GitVersion; -import io.datahub.test.fixtures.elasticsearch.FixtureReader; + import java.util.Optional; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.TestConfiguration; @@ -46,15 +47,15 @@ import java.util.Map; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestConfiguration.REFRESH_INTERVAL_SECONDS; +import static io.datahubproject.test.search.config.SearchTestContainerConfiguration.REFRESH_INTERVAL_SECONDS; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @TestConfiguration -@Import(ESTestConfiguration.class) -public class ESSampleDataFixture { +@Import(SearchCommonTestConfiguration.class) +public class SampleDataFixtureConfiguration { /** * Interested in adding more fixtures? Here's what you will need to update? * 1. Create a new indexPrefix and FixtureName. Both are needed or else all fixtures will load on top of each other, @@ -118,7 +119,7 @@ protected EntityIndexBuilders entityIndexBuilders( @Bean(name = "longTailEntityIndexBuilders") protected EntityIndexBuilders longTailEntityIndexBuilders( - @Qualifier("longTailEntityRegistry") EntityRegistry longTailEntityRegistry, + @Qualifier("entityRegistry") EntityRegistry longTailEntityRegistry, @Qualifier("longTailIndexConvention") IndexConvention indexConvention ) { return entityIndexBuildersHelper(longTailEntityRegistry, indexConvention); @@ -147,7 +148,7 @@ protected ElasticSearchService entitySearchService( @Bean(name = "longTailEntitySearchService") protected ElasticSearchService longTailEntitySearchService( - @Qualifier("longTailEntityRegistry") EntityRegistry longTailEntityRegistry, + @Qualifier("entityRegistry") EntityRegistry longTailEntityRegistry, @Qualifier("longTailEntityIndexBuilders") EntityIndexBuilders longTailEndexBuilders, @Qualifier("longTailIndexConvention") IndexConvention longTailIndexConvention ) throws IOException { @@ -186,7 +187,7 @@ protected SearchService searchService( @Bean(name = "longTailSearchService") @Nonnull protected SearchService longTailSearchService( - @Qualifier("longTailEntityRegistry") EntityRegistry longTailEntityRegistry, + @Qualifier("entityRegistry") EntityRegistry longTailEntityRegistry, @Qualifier("longTailEntitySearchService") ElasticSearchService longTailEntitySearchService, @Qualifier("longTailEntityIndexBuilders") EntityIndexBuilders longTailIndexBuilders, @Qualifier("longTailPrefix") String longTailPrefix, @@ -248,7 +249,7 @@ protected EntityClient entityClient( protected EntityClient longTailEntityClient( @Qualifier("sampleDataSearchService") SearchService searchService, @Qualifier("sampleDataEntitySearchService") ElasticSearchService entitySearchService, - @Qualifier("longTailEntityRegistry") EntityRegistry longTailEntityRegistry + @Qualifier("entityRegistry") EntityRegistry longTailEntityRegistry ) { return entityClientHelper(searchService, entitySearchService, longTailEntityRegistry); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESTestFixtureUtils.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchFixtureUtils.java similarity index 67% rename from metadata-io/src/test/java/com/linkedin/metadata/ESTestFixtureUtils.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchFixtureUtils.java index 914c5be9f5b092..d74dd041f082e8 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESTestFixtureUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchFixtureUtils.java @@ -1,26 +1,45 @@ -package com.linkedin.metadata; +package io.datahubproject.test.fixtures.search; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.datahubproject.test.search.ElasticsearchTestContainer; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; -import io.datahub.test.fixtures.elasticsearch.EntityExporter; -import io.datahub.test.fixtures.elasticsearch.FixtureReader; -import io.datahub.test.fixtures.elasticsearch.FixtureWriter; -import io.datahub.test.fixtures.elasticsearch.LineageExporter; -import io.datahub.test.models.DatasetAnonymized; -import org.elasticsearch.client.RestHighLevelClient; +import io.datahubproject.test.models.DatasetAnonymized; +import io.datahubproject.test.search.SearchTestUtils; +import org.opensearch.client.RestHighLevelClient; import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Bean; +import org.testcontainers.containers.GenericContainer; import org.testng.annotations.Ignore; import org.testng.annotations.Test; import java.io.IOException; import java.util.Set; -import static com.linkedin.metadata.ESTestConfiguration.REFRESH_INTERVAL_SECONDS; -import static com.linkedin.metadata.ESTestUtils.environmentRestClientBuilder; +import static com.linkedin.metadata.Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH; +import static com.linkedin.metadata.Constants.MAX_JACKSON_STRING_SIZE; +/** + * This class is used for extracting and moving search fixture data. + */ @TestConfiguration -@Import(ESTestConfiguration.class) -public class ESTestFixtureUtils { +public class SearchFixtureUtils { + + final public static String FIXTURE_BASE = "src/test/resources/elasticsearch"; + + final public static ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + static { + int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); + OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); + } + + @Bean(name = "testSearchContainer") + public GenericContainer testSearchContainer() { + return new ElasticsearchTestContainer().startContainer(); + } @Test @Ignore("Fixture capture lineage") @@ -37,7 +56,7 @@ private void extractSearchLineageTestFixture() throws IOException { String rootUrn = "urn:li:dataset:(urn:li:dataPlatform:teradata,teradata.simba.pp_bi_tables.tmis_daily_metrics_final_agg,PROD)"; // Set.of("system_metadata_service_v1", "datasetindex_v2", "graph_service_v1") - try (RestHighLevelClient client = new RestHighLevelClient(environmentRestClientBuilder())) { + try (RestHighLevelClient client = new RestHighLevelClient(SearchTestUtils.environmentRestClientBuilder())) { FixtureWriter fixtureWriter = FixtureWriter.builder() .client(client) .build(); @@ -76,7 +95,7 @@ private void extractEntityTestFixture() throws IOException { String prefix = ""; String commonSuffix = "index_v2"; - try (RestHighLevelClient client = new RestHighLevelClient(environmentRestClientBuilder())) { + try (RestHighLevelClient client = new RestHighLevelClient(SearchTestUtils.environmentRestClientBuilder())) { FixtureWriter fixtureWriter = FixtureWriter.builder() .client(client) .build(); @@ -102,7 +121,7 @@ private void extractEntityTestFixture() throws IOException { * 3. Uncomment and run test */ private void reindexTestFixtureData() throws IOException { - ESBulkProcessor bulkProcessor = ESBulkProcessor.builder(new RestHighLevelClient(environmentRestClientBuilder())) + ESBulkProcessor bulkProcessor = ESBulkProcessor.builder(new RestHighLevelClient(SearchTestUtils.environmentRestClientBuilder())) .async(true) .bulkRequestsLimit(1000) .retryInterval(1L) @@ -112,7 +131,7 @@ private void reindexTestFixtureData() throws IOException { FixtureReader reader = FixtureReader.builder() .bulkProcessor(bulkProcessor) .fixtureName("long_tail") - .refreshIntervalSeconds(REFRESH_INTERVAL_SECONDS) + .refreshIntervalSeconds(SearchTestContainerConfiguration.REFRESH_INTERVAL_SECONDS) .build(); reader.read(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java similarity index 95% rename from metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java rename to metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index ade7435bf66523..93d3f108d9e476 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -1,5 +1,7 @@ -package com.linkedin.metadata; +package io.datahubproject.test.fixtures.search; +import io.datahubproject.test.search.config.SearchCommonTestConfiguration; +import io.datahubproject.test.search.config.SearchTestContainerConfiguration; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; @@ -32,9 +34,10 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import com.linkedin.metadata.version.GitVersion; -import io.datahub.test.fixtures.elasticsearch.FixtureReader; + import java.util.Optional; -import org.elasticsearch.client.RestHighLevelClient; + +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.TestConfiguration; @@ -48,12 +51,11 @@ import java.util.Map; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.ESTestConfiguration.REFRESH_INTERVAL_SECONDS; @TestConfiguration -@Import(ESTestConfiguration.class) -public class ESSearchLineageFixture { +@Import(SearchCommonTestConfiguration.class) +public class SearchLineageFixtureConfiguration { @Autowired private ESBulkProcessor _bulkProcessor; @@ -155,7 +157,7 @@ protected LineageSearchService lineageSearchService( .bulkProcessor(_bulkProcessor) .fixtureName(fixtureName) .targetIndexPrefix(prefix) - .refreshIntervalSeconds(REFRESH_INTERVAL_SECONDS) + .refreshIntervalSeconds(SearchTestContainerConfiguration.REFRESH_INTERVAL_SECONDS) .build() .read(); diff --git a/metadata-io/src/test/java/io/datahub/test/models/Anonymized.java b/metadata-io/src/test/java/io/datahubproject/test/models/Anonymized.java similarity index 97% rename from metadata-io/src/test/java/io/datahub/test/models/Anonymized.java rename to metadata-io/src/test/java/io/datahubproject/test/models/Anonymized.java index 1108097dff86a0..60364730630592 100644 --- a/metadata-io/src/test/java/io/datahub/test/models/Anonymized.java +++ b/metadata-io/src/test/java/io/datahubproject/test/models/Anonymized.java @@ -1,4 +1,4 @@ -package io.datahub.test.models; +package io.datahubproject.test.models; import com.fasterxml.jackson.annotation.JsonSetter; diff --git a/metadata-io/src/test/java/io/datahub/test/models/DatasetAnonymized.java b/metadata-io/src/test/java/io/datahubproject/test/models/DatasetAnonymized.java similarity index 97% rename from metadata-io/src/test/java/io/datahub/test/models/DatasetAnonymized.java rename to metadata-io/src/test/java/io/datahubproject/test/models/DatasetAnonymized.java index 225f52d993931f..35813d22067a6e 100644 --- a/metadata-io/src/test/java/io/datahub/test/models/DatasetAnonymized.java +++ b/metadata-io/src/test/java/io/datahubproject/test/models/DatasetAnonymized.java @@ -1,4 +1,4 @@ -package io.datahub.test.models; +package io.datahubproject.test.models; import com.fasterxml.jackson.annotation.JsonGetter; diff --git a/metadata-io/src/test/java/io/datahub/test/models/GraphAnonymized.java b/metadata-io/src/test/java/io/datahubproject/test/models/GraphAnonymized.java similarity index 82% rename from metadata-io/src/test/java/io/datahub/test/models/GraphAnonymized.java rename to metadata-io/src/test/java/io/datahubproject/test/models/GraphAnonymized.java index 5e6c5d57e050ea..3d2360ae042282 100644 --- a/metadata-io/src/test/java/io/datahub/test/models/GraphAnonymized.java +++ b/metadata-io/src/test/java/io/datahubproject/test/models/GraphAnonymized.java @@ -1,4 +1,4 @@ -package io.datahub.test.models; +package io.datahubproject.test.models; import com.fasterxml.jackson.annotation.JsonSetter; @@ -13,7 +13,7 @@ public static class GraphNode extends Anonymized { @JsonSetter("urn") public void setUrn(String urn) { - this.urn = Anonymized.anonymizeUrn(urn); + this.urn = anonymizeUrn(urn); } } } diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/ElasticsearchTestContainer.java b/metadata-io/src/test/java/io/datahubproject/test/search/ElasticsearchTestContainer.java new file mode 100644 index 00000000000000..233a667d078dd2 --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/ElasticsearchTestContainer.java @@ -0,0 +1,42 @@ +package io.datahubproject.test.search; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + + +import static com.linkedin.metadata.DockerTestUtils.checkContainerEngine; + +public class ElasticsearchTestContainer implements SearchTestContainer { + private static final String ELASTIC_VERSION = "7.10.1"; + private static final String ELASTIC_IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch"; + private static final String ENV_ELASTIC_IMAGE_FULL_NAME = System.getenv("ELASTIC_IMAGE_FULL_NAME"); + private static final String ELASTIC_IMAGE_FULL_NAME = ENV_ELASTIC_IMAGE_FULL_NAME != null + ? ENV_ELASTIC_IMAGE_FULL_NAME : ELASTIC_IMAGE_NAME + ":" + ELASTIC_VERSION; + private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse(ELASTIC_IMAGE_FULL_NAME) + .asCompatibleSubstituteFor(ELASTIC_IMAGE_NAME); + + protected static final GenericContainer ES_CONTAINER; + private boolean isStarted = false; + + // A helper method to create an ElasticsearchContainer defaulting to the current image and version, with the ability + // within firewalled environments to override with an environment variable to point to the offline repository. + static { + ES_CONTAINER = new org.testcontainers.elasticsearch.ElasticsearchContainer(DOCKER_IMAGE_NAME); + checkContainerEngine(ES_CONTAINER.getDockerClient()); + ES_CONTAINER.withEnv("ES_JAVA_OPTS", SEARCH_JAVA_OPTS).withStartupTimeout(STARTUP_TIMEOUT); + } + + @Override + public GenericContainer startContainer() { + if (!isStarted) { + ElasticsearchTestContainer.ES_CONTAINER.start(); + isStarted = true; + } + return ES_CONTAINER; + } + + @Override + public void stopContainer() { + ES_CONTAINER.stop(); + } +} diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/OpenSearchTestContainer.java b/metadata-io/src/test/java/io/datahubproject/test/search/OpenSearchTestContainer.java new file mode 100644 index 00000000000000..d94b88b466f893 --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/OpenSearchTestContainer.java @@ -0,0 +1,43 @@ +package io.datahubproject.test.search; + +import org.opensearch.testcontainers.OpensearchContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + + +import static com.linkedin.metadata.DockerTestUtils.checkContainerEngine; + +public class OpenSearchTestContainer implements SearchTestContainer { + private static final String OPENSEARCH_VERSION = "2.9.0"; + private static final String OPENSEARCH_IMAGE_NAME = "opensearchproject/opensearch"; + private static final String ENV_OPENSEARCH_IMAGE_FULL_NAME = System.getenv("OPENSEARCH_IMAGE_FULL_NAME"); + private static final String OPENSEARCH_IMAGE_FULL_NAME = ENV_OPENSEARCH_IMAGE_FULL_NAME != null + ? ENV_OPENSEARCH_IMAGE_FULL_NAME : OPENSEARCH_IMAGE_NAME + ":" + OPENSEARCH_VERSION; + private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse(OPENSEARCH_IMAGE_FULL_NAME) + .asCompatibleSubstituteFor(OPENSEARCH_IMAGE_NAME); + + protected static final GenericContainer OS_CONTAINER; + private boolean isStarted = false; + + // A helper method to create an ElasticseachContainer defaulting to the current image and version, with the ability + // within firewalled environments to override with an environment variable to point to the offline repository. + static { + OS_CONTAINER = new OpensearchContainer(DOCKER_IMAGE_NAME); + checkContainerEngine(OS_CONTAINER.getDockerClient()); + OS_CONTAINER.withEnv("OPENSEARCH_JAVA_OPTS", SEARCH_JAVA_OPTS).withStartupTimeout(STARTUP_TIMEOUT); + } + + @Override + public GenericContainer startContainer() { + if (!isStarted) { + OS_CONTAINER.start(); + isStarted = true; + } + return OS_CONTAINER; + } + + @Override + public void stopContainer() { + OS_CONTAINER.stop(); + } +} diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java new file mode 100644 index 00000000000000..67e1ee368f5136 --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java @@ -0,0 +1,14 @@ +package io.datahubproject.test.search; + +import org.testcontainers.containers.GenericContainer; + +import java.time.Duration; + +public interface SearchTestContainer { + String SEARCH_JAVA_OPTS = "-Xms64m -Xmx384m -XX:MaxDirectMemorySize=368435456"; + Duration STARTUP_TIMEOUT = Duration.ofMinutes(5); // usually < 1min + + GenericContainer startContainer(); + + void stopContainer(); +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java similarity index 74% rename from metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java rename to metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java index 7e9605cbe3db08..414b9f927fada9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata; +package io.datahubproject.test.search; import com.datahub.authentication.Authentication; import com.datahub.plugins.auth.authorization.Authorizer; @@ -17,48 +17,32 @@ import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.annotation.Nullable; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.utility.DockerImageName; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.AUTO_COMPLETE_ENTITY_TYPES; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES; -import static com.linkedin.metadata.DockerTestUtils.checkContainerEngine; -public class ESTestUtils { - private ESTestUtils() { +public class SearchTestUtils { + private SearchTestUtils() { } - private static final String ELASTIC_VERSION = "7.10.1"; - private static final String ELASTIC_IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch"; - private static final String ENV_ELASTIC_IMAGE_FULL_NAME = System.getenv("ELASTIC_IMAGE_FULL_NAME"); - private static final String ELASTIC_IMAGE_FULL_NAME = ENV_ELASTIC_IMAGE_FULL_NAME != null - ? ENV_ELASTIC_IMAGE_FULL_NAME : ELASTIC_IMAGE_NAME + ":" + ELASTIC_VERSION; - private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse(ELASTIC_IMAGE_FULL_NAME) - .asCompatibleSubstituteFor(ELASTIC_IMAGE_NAME); - - public static final ElasticsearchContainer ES_CONTAINER; - - // A helper method to create an ElasticseachContainer defaulting to the current image and version, with the ability - // within firewalled environments to override with an environment variable to point to the offline repository. - static { - ES_CONTAINER = new ElasticsearchContainer(DOCKER_IMAGE_NAME); - checkContainerEngine(ES_CONTAINER.getDockerClient()); - ES_CONTAINER.withEnv("ES_JAVA_OPTS", "-Xms64m -Xmx384m -XX:MaxDirectMemorySize=368435456") - .withStartupTimeout(Duration.ofMinutes(5)); // usually < 1min + public static void syncAfterWrite(ESBulkProcessor bulkProcessor) throws InterruptedException { + bulkProcessor.flush(); + Thread.sleep(1000); } public final static List SEARCHABLE_ENTITIES; @@ -75,7 +59,7 @@ public static SearchResult searchAcrossEntities(SearchService searchService, Str public static SearchResult searchAcrossEntities(SearchService searchService, String query, @Nullable List facets) { return searchService.searchAcrossEntities(SEARCHABLE_ENTITIES, query, null, null, 0, - 100, new SearchFlags().setFulltext(true).setSkipCache(true), facets); + 100, new SearchFlags().setFulltext(true).setSkipCache(true), facets); } public static SearchResult searchAcrossCustomEntities(SearchService searchService, String query, List searchableEntities) { @@ -89,12 +73,12 @@ public static SearchResult search(SearchService searchService, String query) { public static SearchResult search(SearchService searchService, List entities, String query) { return searchService.search(entities, query, null, null, 0, 100, - new SearchFlags().setFulltext(true).setSkipCache(true)); + new SearchFlags().setFulltext(true).setSkipCache(true)); } public static ScrollResult scroll(SearchService searchService, String query, int batchSize, @Nullable String scrollId) { return searchService.scrollAcrossEntities(SEARCHABLE_ENTITIES, query, null, null, - scrollId, "3m", batchSize, new SearchFlags().setFulltext(true).setSkipCache(true)); + scrollId, "3m", batchSize, new SearchFlags().setFulltext(true).setSkipCache(true)); } public static SearchResult searchStructured(SearchService searchService, String query) { @@ -112,9 +96,9 @@ public static LineageSearchResult lineage(LineageSearchService lineageSearchServ .build()); return lineageSearchService.searchAcrossLineage(root, LineageDirection.DOWNSTREAM, - SEARCHABLE_ENTITY_TYPES.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()), - "*", hops, ResolverUtils.buildFilter(filters, List.of()), null, 0, 100, null, - null, new SearchFlags().setSkipCache(true)); + SEARCHABLE_ENTITY_TYPES.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()), + "*", hops, ResolverUtils.buildFilter(filters, List.of()), null, 0, 100, null, + null, new SearchFlags().setSkipCache(true)); } public static AutoCompleteResults autocomplete(SearchableEntityType searchableEntityType, String query) throws Exception { @@ -160,4 +144,4 @@ public HttpAsyncClientBuilder customizeHttpClient( } }); } -} \ No newline at end of file +} diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java new file mode 100644 index 00000000000000..530d3f4d536259 --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java @@ -0,0 +1,63 @@ +package io.datahubproject.test.search.config; + +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import com.linkedin.metadata.config.search.CustomConfiguration; +import com.linkedin.metadata.config.search.ExactMatchConfiguration; +import com.linkedin.metadata.config.search.PartialConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.WordGramConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistryException; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; + +/** + * This is common configuration for search regardless of which + * test container implementation. + */ +@TestConfiguration +public class SearchCommonTestConfiguration { + @Bean + public SearchConfiguration searchConfiguration() { + SearchConfiguration searchConfiguration = new SearchConfiguration(); + searchConfiguration.setMaxTermBucketSize(20); + + ExactMatchConfiguration exactMatchConfiguration = new ExactMatchConfiguration(); + exactMatchConfiguration.setExclusive(false); + exactMatchConfiguration.setExactFactor(10.0f); + exactMatchConfiguration.setWithPrefix(true); + exactMatchConfiguration.setPrefixFactor(6.0f); + exactMatchConfiguration.setCaseSensitivityFactor(0.7f); + exactMatchConfiguration.setEnableStructured(true); + + WordGramConfiguration wordGramConfiguration = new WordGramConfiguration(); + wordGramConfiguration.setTwoGramFactor(1.2f); + wordGramConfiguration.setThreeGramFactor(1.5f); + wordGramConfiguration.setFourGramFactor(1.8f); + + PartialConfiguration partialConfiguration = new PartialConfiguration(); + partialConfiguration.setFactor(0.4f); + partialConfiguration.setUrnFactor(0.5f); + + searchConfiguration.setExactMatch(exactMatchConfiguration); + searchConfiguration.setWordGram(wordGramConfiguration); + searchConfiguration.setPartial(partialConfiguration); + return searchConfiguration; + } + + @Bean + public CustomSearchConfiguration customSearchConfiguration() throws Exception { + CustomConfiguration customConfiguration = new CustomConfiguration(); + customConfiguration.setEnabled(true); + customConfiguration.setFile("search_config_builder_test.yml"); + return customConfiguration.resolve(new YAMLMapper()); + } + + @Bean(name = "entityRegistry") + public EntityRegistry entityRegistry() throws EntityRegistryException { + return new ConfigEntityRegistry( + SearchCommonTestConfiguration.class.getClassLoader().getResourceAsStream("entity-registry.yml")); + } +} diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java new file mode 100644 index 00000000000000..2cfa9f91878258 --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java @@ -0,0 +1,88 @@ +package io.datahubproject.test.search.config; + +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import com.linkedin.metadata.version.GitVersion; +import java.util.Optional; + +import org.apache.http.HttpHost; +import org.apache.http.impl.nio.reactor.IOReactorConfig; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.testcontainers.containers.GenericContainer; + +import javax.annotation.Nonnull; + +import java.util.Map; + + +/** + * This configuration is for `test containers` it builds these objects tied to + * the test container instantiated for tests. Could be ES or OpenSearch, etc. + * + * Does your test required a running instance? If no, {@link io.datahubproject.test.search.config.SearchCommonTestConfiguration} instead. + */ +@TestConfiguration +public class SearchTestContainerConfiguration { + // This port is overridden by the specific test container instance + private static final int HTTP_PORT = 9200; + public static final int REFRESH_INTERVAL_SECONDS = 5; + + @Primary + @Bean(name = "searchRestHighLevelClient") + @Nonnull + public RestHighLevelClient getElasticsearchClient(@Qualifier("testSearchContainer") GenericContainer searchContainer) { + // A helper method to create a search test container defaulting to the current image and version, with the ability + // within firewalled environments to override with an environment variable to point to the offline repository. + // A helper method to construct a standard rest client for search. + final RestClientBuilder builder = + RestClient.builder(new HttpHost( + "localhost", + searchContainer.getMappedPort(HTTP_PORT), "http") + ).setHttpClientConfigCallback(httpAsyncClientBuilder -> + httpAsyncClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build())); + + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder. + setConnectionRequestTimeout(30000)); + + return new RestHighLevelClient(builder); + } + + /* + Cannot use the factory class without circular dependencies + */ + @Primary + @Bean(name = "searchBulkProcessor") + @Nonnull + public ESBulkProcessor getBulkProcessor(@Qualifier("searchRestHighLevelClient") RestHighLevelClient searchClient) { + return ESBulkProcessor.builder(searchClient) + .async(true) + /* + * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful + * to present a consistent view to for indices with very low traffic. And it is wonderful for tests! + */ + .writeRequestRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .bulkRequestsLimit(10000) + .bulkFlushPeriod(REFRESH_INTERVAL_SECONDS - 1) + .retryInterval(1L) + .numRetries(1) + .build(); + } + + @Primary + @Bean(name = "searchIndexBuilder") + @Nonnull + protected ESIndexBuilder getIndexBuilder(@Qualifier("searchRestHighLevelClient") RestHighLevelClient searchClient) { + GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); + return new ESIndexBuilder(searchClient, 1, 1, 3, 1, Map.of(), + false, false, + new ElasticSearchConfiguration(), gitVersion); + } +} diff --git a/metadata-io/src/test/resources/testng-other.xml b/metadata-io/src/test/resources/testng-other.xml new file mode 100644 index 00000000000000..e214fdb8c1f61f --- /dev/null +++ b/metadata-io/src/test/resources/testng-other.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/metadata-io/src/test/resources/testng-search.xml b/metadata-io/src/test/resources/testng-search.xml new file mode 100644 index 00000000000000..3b32ae34c1f5ad --- /dev/null +++ b/metadata-io/src/test/resources/testng-search.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/metadata-io/src/test/resources/testng.xml b/metadata-io/src/test/resources/testng.xml new file mode 100644 index 00000000000000..fdd1c1a6c8921e --- /dev/null +++ b/metadata-io/src/test/resources/testng.xml @@ -0,0 +1,14 @@ + + + + + + + + + \ No newline at end of file diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index 69fe2255a69165..d36fd0de40d035 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -44,6 +44,7 @@ dependencies { testImplementation externalDependency.mockito implementation externalDependency.awsMskIamAuth + testImplementation externalDependency.testng testImplementation externalDependency.springBootTest testRuntimeOnly externalDependency.logbackClassic } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java index 7ba04ecd2389e8..b0fade24e26ada 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java @@ -2,7 +2,7 @@ import com.linkedin.events.metadata.ChangeType; import lombok.Data; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; @Data public abstract class ElasticEvent { diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java index afa69c9f1750e2..bea75f7b282ee1 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java @@ -5,11 +5,11 @@ import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.common.xcontent.XContentType; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.common.xcontent.XContentType; @Slf4j diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java index d42464051d7ec7..230cd8433e6ffd 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java @@ -1,11 +1,11 @@ package com.linkedin.metadata.kafka.elasticsearch; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import javax.annotation.Nullable; diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java index 184efa1573b359..a3d6dca75068be 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java @@ -2,12 +2,12 @@ import com.linkedin.data.template.RecordTemplate; import com.datahub.util.RecordUtils; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import javax.annotation.Nullable; diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index db01be3ccebdfc..53e7765152aefe 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -34,6 +34,7 @@ dependencies { swaggerCodegen externalDependency.swaggerCli testImplementation externalDependency.guava + testImplementation externalDependency.testngJava8 } sourceSets { diff --git a/metadata-service/auth-impl/build.gradle b/metadata-service/auth-impl/build.gradle index 1ffeb99e7ad4af..60d622dea54475 100644 --- a/metadata-service/auth-impl/build.gradle +++ b/metadata-service/auth-impl/build.gradle @@ -24,4 +24,5 @@ dependencies { annotationProcessor externalDependency.lombok testImplementation externalDependency.mockito + testImplementation externalDependency.testng } \ No newline at end of file diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java index 1da66f3192f807..5f50b8f7f05083 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java @@ -24,9 +24,9 @@ import org.apache.http.nio.reactor.IOReactorException; import org.apache.http.nio.reactor.IOReactorExceptionHandler; import org.apache.http.ssl.SSLContexts; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index 0be69e5dad58d9..d7aee59ca6dd18 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -39,7 +39,7 @@ import com.linkedin.metadata.version.GitVersion; import com.linkedin.usage.UsageClient; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java index c74f5e11cadce9..c266b3635b16ff 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java @@ -7,7 +7,7 @@ import com.linkedin.metadata.recommendation.candidatesource.MostPopularSource; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java index 58584a4d957de5..109cc8dbc82d1b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java @@ -7,7 +7,7 @@ import com.linkedin.metadata.recommendation.candidatesource.RecentlyEditedSource; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlySearchedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlySearchedCandidateSourceFactory.java index b3779a132284f7..5209f65a2ec63b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlySearchedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlySearchedCandidateSourceFactory.java @@ -5,7 +5,7 @@ import com.linkedin.metadata.recommendation.candidatesource.RecentlySearchedSource; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java index d0505e8d2a3eaa..aea40b4d8eb463 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java @@ -7,7 +7,7 @@ import com.linkedin.metadata.recommendation.candidatesource.RecentlyViewedSource; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java index eeb32ae1ddbf9f..620af803723e7a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java @@ -8,7 +8,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Value; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java index 956157f70e6bc3..fc6f92b2678f3f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java @@ -6,8 +6,8 @@ import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java index decbc2e12a9988..495d77ccbb29f4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java @@ -13,7 +13,7 @@ import javax.annotation.Nullable; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java index 2972316856a8d0..2610ebd3528cda 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java @@ -12,7 +12,7 @@ import java.io.IOException; import java.util.Optional; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.joda.time.DateTime; import org.json.JSONObject; import org.springframework.scheduling.annotation.Scheduled; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java index c5501067ff3932..7cdca996a81315 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java @@ -6,7 +6,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.version.GitVersion; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java index 859c8e18cacffc..266039afb45d5c 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java @@ -2,7 +2,7 @@ import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; -import org.elasticsearch.action.support.WriteRequest; +import org.opensearch.action.support.WriteRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.context.SpringBootTest; diff --git a/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java b/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java index 28c47f169a111a..fe0d61986b4a61 100644 --- a/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java +++ b/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java @@ -7,8 +7,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; +import static org.testng.AssertJUnit.assertEquals; public class TelemetryUtilsTest { diff --git a/metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java b/metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java index 02ca5182cd2bec..c200e63e0d4977 100644 --- a/metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java +++ b/metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java @@ -11,11 +11,11 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.cluster.health.ClusterHealthStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpStatus; diff --git a/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java b/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java index 98f0db8fd10efd..83b1b3f87c7249 100644 --- a/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java +++ b/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java @@ -7,7 +7,7 @@ import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.AuthorizerChain; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; -import org.elasticsearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponse; import org.mockito.Mockito; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java index 8f87b041a7e036..e0fec074523027 100644 --- a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java @@ -17,8 +17,8 @@ import java.util.List; -import static org.junit.Assert.assertNotNull; import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; @Import({OpenAPIEntityTestConfiguration.class}) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java index 7910982a63133c..f29461734ebfc7 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java @@ -13,7 +13,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.tasks.GetTaskResponse; +import org.opensearch.client.tasks.GetTaskResponse; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java index 17de9ceea35a30..1e6523e774d66c 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java @@ -35,7 +35,7 @@ import javax.inject.Inject; import javax.inject.Named; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.tasks.GetTaskResponse; +import org.opensearch.client.tasks.GetTaskResponse; import org.json.JSONObject; import static com.linkedin.metadata.Constants.*; diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java index 665bc3cfc277ce..470c6e87040ec5 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java +++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java @@ -3,12 +3,13 @@ import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.util.Pair; import java.util.List; -import junit.framework.TestCase; import mock.MockTimeseriesAspectService; import org.testng.annotations.Test; +import static org.testng.AssertJUnit.*; -public class OperationsResourceTest extends TestCase { + +public class OperationsResourceTest { private static final String TASK_ID = "taskId123"; diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle index 99345d6f6bc3f3..22c62af324c12d 100644 --- a/metadata-service/services/build.gradle +++ b/metadata-service/services/build.gradle @@ -63,8 +63,6 @@ dependencies { } test { - // https://docs.gradle.org/current/userguide/performance.html - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 testLogging.showStandardStreams = true testLogging.exceptionFormat = 'full' } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java index ac17c882c24b69..357a5df2edd444 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java @@ -20,18 +20,18 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java index e6f2106bd5c3e4..ea59885e8b6d53 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java @@ -8,7 +8,7 @@ import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.elasticsearch.client.tasks.GetTaskResponse; +import org.opensearch.client.tasks.GetTaskResponse; public interface SystemMetadataService { diff --git a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java index 2c26c00e9c4d64..d788222c5d87b3 100644 --- a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java +++ b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java @@ -9,16 +9,16 @@ import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.search.elasticsearch.query.request.SearchRequestHandler; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.SimpleQueryStringBuilder; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.WeightBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.opensearch.index.query.functionscore.WeightBuilder; import org.springframework.web.context.WebApplicationContext; import org.springframework.web.context.support.WebApplicationContextUtils; diff --git a/metadata-service/servlet/src/main/java/com/datahub/gms/util/CSVWriter.java b/metadata-service/servlet/src/main/java/com/datahub/gms/util/CSVWriter.java index e9d1308e857d88..79d4f7077b7978 100644 --- a/metadata-service/servlet/src/main/java/com/datahub/gms/util/CSVWriter.java +++ b/metadata-service/servlet/src/main/java/com/datahub/gms/util/CSVWriter.java @@ -2,8 +2,8 @@ import lombok.Builder; -import org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder; -import org.elasticsearch.index.query.functionscore.WeightBuilder; +import org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder; +import org.opensearch.index.query.functionscore.WeightBuilder; import java.io.PrintWriter; import java.util.stream.Collectors; diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle index 9f8ef70a0e7284..1c1c368611488f 100644 --- a/metadata-utils/build.gradle +++ b/metadata-utils/build.gradle @@ -26,6 +26,7 @@ dependencies { testImplementation project(':test-models') testImplementation project(path: ':test-models', configuration: 'testDataTemplate') + testImplementation externalDependency.testng constraints { implementation(externalDependency.log4jCore) { diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java index 8b4b500dfc4552..69bd3b461eb12b 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java @@ -18,8 +18,8 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh index 41dca90acc9fc1..93f03d36cbd190 100755 --- a/smoke-test/cypress-dev.sh +++ b/smoke-test/cypress-dev.sh @@ -17,4 +17,5 @@ npm install source ../../set-cypress-creds.sh -npx cypress open +npx cypress open \ + --env "ADMIN_DISPLAYNAME=$CYPRESS_ADMIN_DISPLAYNAME,ADMIN_USERNAME=$CYPRESS_ADMIN_USERNAME,ADMIN_PASSWORD=$CYPRESS_ADMIN_PASSWORD" diff --git a/smoke-test/run-quickstart.sh b/smoke-test/run-quickstart.sh index 050b5d2db95c95..cd747321ad6023 100755 --- a/smoke-test/run-quickstart.sh +++ b/smoke-test/run-quickstart.sh @@ -12,7 +12,16 @@ pip install -r requirements.txt mkdir -p ~/.datahub/plugins/frontend/auth/ echo "test_user:test_pass" >> ~/.datahub/plugins/frontend/auth/user.props +DATAHUB_SEARCH_IMAGE="${DATAHUB_SEARCH_IMAGE:=opensearchproject/opensearch}" +DATAHUB_SEARCH_TAG="${DATAHUB_SEARCH_TAG:=2.9.0}" +XPACK_SECURITY_ENABLED="${XPACK_SECURITY_ENABLED:=plugins.security.disabled=true}" +ELASTICSEARCH_USE_SSL="${ELASTICSEARCH_USE_SSL:=false}" +USE_AWS_ELASTICSEARCH="${USE_AWS_ELASTICSEARCH:=true}" + echo "DATAHUB_VERSION = $DATAHUB_VERSION" DATAHUB_TELEMETRY_ENABLED=false \ DOCKER_COMPOSE_BASE="file://$( dirname "$DIR" )" \ +DATAHUB_SEARCH_IMAGE="$DATAHUB_SEARCH_IMAGE" DATAHUB_SEARCH_TAG="$DATAHUB_SEARCH_TAG" \ +XPACK_SECURITY_ENABLED="$XPACK_SECURITY_ENABLED" ELASTICSEARCH_USE_SSL="$ELASTICSEARCH_USE_SSL" \ +USE_AWS_ELASTICSEARCH="$USE_AWS_ELASTICSEARCH" \ datahub docker quickstart --version ${DATAHUB_VERSION} --standalone_consumers --dump-logs-on-failure --kafka-setup diff --git a/smoke-test/set-cypress-creds.sh b/smoke-test/set-cypress-creds.sh index 0512724e9a269d..82fe736b0a7e18 100644 --- a/smoke-test/set-cypress-creds.sh +++ b/smoke-test/set-cypress-creds.sh @@ -1,4 +1,5 @@ #!/bin/bash export CYPRESS_ADMIN_USERNAME=${ADMIN_USERNAME:-datahub} -export CYPRESS_ADMIN_PASSWORD=${ADMIN_PASSWORD:-datahub} \ No newline at end of file +export CYPRESS_ADMIN_PASSWORD=${ADMIN_PASSWORD:-datahub} +export CYPRESS_ADMIN_DISPLAYNAME=${ADMIN_DISPLAYNAME:-DataHub} \ No newline at end of file diff --git a/smoke-test/tests/cypress/cypress/e2e/login/login.js b/smoke-test/tests/cypress/cypress/e2e/login/login.js index f86741b5afe010..309eedb10b6da9 100644 --- a/smoke-test/tests/cypress/cypress/e2e/login/login.js +++ b/smoke-test/tests/cypress/cypress/e2e/login/login.js @@ -4,6 +4,6 @@ describe('login', () => { cy.get('input[data-testid=username]').type(Cypress.env('ADMIN_USERNAME')); cy.get('input[data-testid=password]').type(Cypress.env('ADMIN_PASSWORD')); cy.contains('Sign In').click(); - cy.contains('Welcome back, DataHub'); + cy.contains('Welcome back, ' + Cypress.env('ADMIN_DISPLAYNAME')); }); })