Merge branch 'feat/mysqlUpgrade' of https://github.com/RyanHolstien/d…

…atahub into feat/mysqlUpgrade
datahub-project · Nov 17, 2023 · 9b7d78d · 9b7d78d
2 parents ee2d129 + 10979c8
commit 9b7d78d
Show file tree

Hide file tree

Showing 66 changed files with 938 additions and 2,284 deletions.
diff --git a/build.gradle b/build.gradle
@@ -19,7 +19,7 @@ buildscript {
   ext.logbackClassic = '1.2.12'
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '2.3.0'
-  ext.hazelcastVersion = '5.3.1'
+  ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
 
   ext.docker_registry = 'linkedin'
@@ -53,7 +53,7 @@ project.ext.spec = [
         'pegasus' : [
             'd2' : 'com.linkedin.pegasus:d2:' + pegasusVersion,
             'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
-            'dataAvro1_6' : 'com.linkedin.pegasus:data-avro-1_6:' + pegasusVersion,
+            'dataAvro': 'com.linkedin.pegasus:data-avro:' + pegasusVersion,
             'generator': 'com.linkedin.pegasus:generator:' + pegasusVersion,
             'restliCommon' : 'com.linkedin.pegasus:restli-common:' + pegasusVersion,
             'restliClient' : 'com.linkedin.pegasus:restli-client:' + pegasusVersion,
@@ -71,22 +71,22 @@ project.ext.externalDependency = [
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
-    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
-    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
-    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
-    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0',
+    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
+    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.9',
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
     'awsRds':'software.amazon.awssdk:rds:2.18.24',
-    'cacheApi' : 'javax.cache:cache-api:1.1.0',
+    'cacheApi': 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
     'commonsText': 'org.apache.commons:commons-text:1.10.0',
     'commonsCollections': 'commons-collections:commons-collections:3.2.2',
-    'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
+    'caffeine': 'com.github.ben-manes.caffeine:caffeine:3.1.8',
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
     'datastaxOssCore': 'com.datastax.oss:java-driver-core:4.14.1',
     'datastaxOssQueryBuilder': 'com.datastax.oss:java-driver-query-builder:4.14.1',
-    'dgraph4j' : 'io.dgraph:dgraph4j:21.03.1',
+    'dgraph4j' : 'io.dgraph:dgraph4j:21.12.0',
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
@@ -131,7 +131,7 @@ project.ext.externalDependency = [
     'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
-    'json': 'org.json:json:20230227',
+    'json': 'org.json:json:20231013',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
@@ -140,7 +140,7 @@ project.ext.externalDependency = [
     'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
     'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion",
-    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.3',
+    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.4',
     'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
     'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
     'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
@@ -164,6 +164,7 @@ project.ext.externalDependency = [
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
+    'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
     'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
@@ -178,6 +179,7 @@ project.ext.externalDependency = [
     'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
     'postgresql': 'org.postgresql:postgresql:42.3.8',
     'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
+    'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.9',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
@@ -201,7 +203,7 @@ project.ext.externalDependency = [
     'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion",
     'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion",
     'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
-    'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11',
+    'springKafka': 'org.springframework.kafka:spring-kafka:2.9.13',
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
@@ -263,7 +265,7 @@ subprojects {
   plugins.withType(JavaPlugin) {
     dependencies {
       constraints {
-        implementation('io.netty:netty-all:4.1.86.Final')
+        implementation('io.netty:netty-all:4.1.100.Final')
         implementation('org.apache.commons:commons-compress:1.21')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')

diff --git a/...ava/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/...ava/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java
@@ -79,11 +79,11 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
     }
 
     // 2. Get list of entities that we should query based on filters or assets from aspect.
-    List<String> entitiesToQuery = assetUrns.stream().map(Urn::getEntityType).collect(Collectors.toList());
+    List<String> entitiesToQuery = assetUrns.stream().map(Urn::getEntityType).distinct().collect(Collectors.toList());
 
 
     final List<EntityType> inputEntityTypes = (input.getTypes() == null || input.getTypes().isEmpty()) ? ImmutableList.of() : input.getTypes();
-    final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList());
+    final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).distinct().collect(Collectors.toList());
 
     final List<String> finalEntityNames = inputEntityNames.size() > 0 ? inputEntityNames : entitiesToQuery;
 

diff --git a/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
@@ -70,7 +70,7 @@ public static List<MatchedField> getMatchedFieldEntry(List<com.linkedin.metadata
                   Urn urn = Urn.createFromString(field.getValue());
                   matchedField.setEntity(UrnToEntityMapper.map(urn));
               } catch (URISyntaxException e) {
-                  log.warn("Failed to create urn from MatchedField value: {}", field.getValue(), e);
+                  log.debug("Failed to create urn from MatchedField value: {}", field.getValue());
               }
           }
           return matchedField;

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -66,7 +66,9 @@ dependencies {
   runtimeOnly externalDependency.mysqlConnector
   runtimeOnly externalDependency.postgresql
 
-  implementation externalDependency.awsMskIamAuth
+  implementation(externalDependency.awsMskIamAuth) {
+    exclude group: 'software.amazon.awssdk', module: 'third-party-jackson-core'
+  }
 
   annotationProcessor externalDependency.lombok
   annotationProcessor externalDependency.picocli
@@ -75,6 +77,12 @@ dependencies {
   testImplementation externalDependency.mockito
   testImplementation externalDependency.testng
   testRuntimeOnly externalDependency.logbackClassic
+
+  constraints {
+    implementation(implementation externalDependency.parquetHadoop) {
+      because("CVE-2022-42003")
+    }
+  }
 }
 
 bootJar {

diff --git a/datahub-web-react/.env b/datahub-web-react/.env
@@ -1,4 +1,5 @@
 PUBLIC_URL=/assets
 REACT_APP_THEME_CONFIG=theme_light.config.json
 SKIP_PREFLIGHT_CHECK=true
-BUILD_PATH=build/yarn
+BUILD_PATH=build/yarn
+REACT_APP_PROXY_TARGET=http://localhost:9002
diff --git a/datahub-web-react/README.md b/datahub-web-react/README.md
@@ -51,6 +51,14 @@ need to be deployed, still at `http://localhost:9002`, to service GraphQL API re
 
 Optionally you could also start the app with the mock server without running the docker containers by executing `yarn start:mock`. See [here](src/graphql-mock/fixtures/searchResult/userSearchResult.ts#L6) for available login users.
 
+### Testing your customizations
+
+There is two options to test your customizations:
+* **Option 1**: Initialize the docker containers with the `quickstart.sh` script (or if any custom docker-compose file) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at `http://localhost:9002` to fetch real data.
+* **Option 2**: Change the environment variable `REACT_APP_PROXY_TARGET` in the `.env` file to point to your `datahub-frontend` server (ex: https://my_datahub_host.com) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at some domain to fetch real data.
+
+The option 2 is useful if you want to test your React customizations without having to run the hole DataHub stack locally. However, if you changed other components of the DataHub stack, you will need to run the hole stack locally (building the docker images) and use the option 1.
+
 ### Functional testing
 
 In order to start a server and run frontend unit tests using react-testing-framework, run:

diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle
@@ -16,10 +16,10 @@ node {
   }
 
   // Version of node to use.
-  version = '16.8.0'
+  version = '21.2.0'
 
   // Version of Yarn to use.
-  yarnVersion = '1.22.0'
+  yarnVersion = '1.22.1'
 
   // Base URL for fetching node distributions (set nodeDistBaseUrl if you have a mirror).
   if (project.hasProperty('nodeDistBaseUrl')) {

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
@@ -95,7 +95,7 @@
         "start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start",
         "start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start",
         "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start",
-        "build": "yarn run generate && CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/",
+        "build": "yarn run generate && NODE_OPTIONS=--openssl-legacy-provider CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/",
         "test": "craco test",
         "pretest:e2e:ci": "yarn generate",
         "test:e2e": "start-server-and-test start:e2e 3010",

diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx
@@ -1,17 +1,12 @@
-import React, { CSSProperties, useRef, useState } from 'react';
-import { Link } from 'react-router-dom';
+import React, { useRef, useState } from 'react';
+import { LoadingOutlined } from '@ant-design/icons';
 import styled from 'styled-components/macro';
-import Highlight from 'react-highlighter';
 import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated';
 import { EntityType } from '../../types.generated';
-import { IconStyleType } from '../entity/Entity';
-import { ANTD_GRAY } from '../entity/shared/constants';
 import { SearchBar } from '../search/SearchBar';
 import ClickOutside from '../shared/ClickOutside';
 import { useEntityRegistry } from '../useEntityRegistry';
-import DomainIcon from './DomainIcon';
-import ParentEntities from '../search/filters/ParentEntities';
-import { getParentDomains } from './utils';
+import DomainSearchResultItem from './DomainSearchResultItem';
 
 const DomainSearchWrapper = styled.div`
     position: relative;
@@ -33,34 +28,19 @@ const ResultsWrapper = styled.div`
     z-index: 1;
 `;
 
-const SearchResult = styled(Link)`
-    color: #262626;
+const LoadingWrapper = styled.div`
     display: flex;
     align-items: center;
-    gap: 8px;
-    height: 100%;
-    padding: 6px 8px;
-    width: 100%;
-    &:hover {
-        background-color: ${ANTD_GRAY[3]};
-        color: #262626;
-    }
+    justify-content: center;
+    height: 350px;
+    font-size: 30px;
 `;
 
-const IconWrapper = styled.span``;
-
-const highlightMatchStyle: CSSProperties = {
-    fontWeight: 'bold',
-    background: 'none',
-    padding: 0,
-};
-
 function DomainSearch() {
     const [query, setQuery] = useState('');
     const [isSearchBarFocused, setIsSearchBarFocused] = useState(false);
     const entityRegistry = useEntityRegistry();
-
-    const { data } = useGetSearchResultsForMultipleQuery({
+    const { data, loading } = useGetSearchResultsForMultipleQuery({
         variables: {
             input: {
                 types: [EntityType.Domain],
@@ -69,18 +49,38 @@ function DomainSearch() {
                 count: 50,
             },
         },
-        skip: !query,
     });
 
     const searchResults = data?.searchAcrossEntities?.searchResults;
     const timerRef = useRef(-1);
+
     const handleQueryChange = (q: string) => {
         window.clearTimeout(timerRef.current);
         timerRef.current = window.setTimeout(() => {
             setQuery(q);
         }, 250);
     };
 
+    const renderLoadingIndicator = () => (
+        <LoadingWrapper>
+            <LoadingOutlined />
+        </LoadingWrapper>
+    );
+
+    const renderSearchResults = () => (
+        <ResultsWrapper>
+            {searchResults?.map((result) => (
+                <DomainSearchResultItem
+                    key={result.entity.urn}
+                    entity={result.entity}
+                    entityRegistry={entityRegistry}
+                    query={query}
+                    onResultClick={() => setIsSearchBarFocused(false)}
+                />
+            ))}
+        </ResultsWrapper>
+    );
+
     return (
         <DomainSearchWrapper>
             <ClickOutside onClickOutside={() => setIsSearchBarFocused(false)}>
@@ -102,39 +102,8 @@ function DomainSearch() {
                     entityRegistry={entityRegistry}
                     onFocus={() => setIsSearchBarFocused(true)}
                 />
-                {isSearchBarFocused && searchResults && !!searchResults.length && (
-                    <ResultsWrapper>
-                        {searchResults.map((result) => {
-                            return (
-                                <SearchResult
-                                    to={entityRegistry.getEntityUrl(result.entity.type, result.entity.urn)}
-                                    onClick={() => setIsSearchBarFocused(false)}
-                                >
-                                    <IconWrapper>
-                                        {result.entity.type === EntityType.Domain ? (
-                                            <DomainIcon
-                                                style={{
-                                                    fontSize: 16,
-                                                    color: '#BFBFBF',
-                                                }}
-                                            />
-                                        ) : (
-                                            entityRegistry.getIcon(result.entity.type, 12, IconStyleType.ACCENT)
-                                        )}
-                                    </IconWrapper>
-                                    <div>
-                                        <ParentEntities
-                                            parentEntities={getParentDomains(result.entity, entityRegistry)}
-                                        />
-                                        <Highlight matchStyle={highlightMatchStyle} search={query}>
-                                            {entityRegistry.getDisplayName(result.entity.type, result.entity)}
-                                        </Highlight>
-                                    </div>
-                                </SearchResult>
-                            );
-                        })}
-                    </ResultsWrapper>
-                )}
+                {loading && renderLoadingIndicator()}
+                {!loading && isSearchBarFocused && !!searchResults?.length && renderSearchResults()}
             </ClickOutside>
         </DomainSearchWrapper>
     );