Merge branch 'master' into fix/metadata-io/use-proper-algorithm-to-ge…

…t-lineage
datahub-project · Oct 30, 2023 · 503291f · 503291f
2 parents ce1ece3 + f5c8192
commit 503291f
Show file tree

Hide file tree

Showing 72 changed files with 3,844 additions and 1,739 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -12,7 +12,7 @@ on:
     branches:
       - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/airflow-plugin.yml"
       - "metadata-ingestion-modules/airflow-plugin/**"
       - "metadata-ingestion/**"
       - "metadata-models/**"

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -4,17 +4,13 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docker/**"
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-integration"
   pull_request:
     branches:
       - "**"
-    paths-ignore:
-      - "docker/**"
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-integration"
   release:
     types: [published]
 

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -851,8 +851,14 @@ jobs:
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
-          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
+          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -4,9 +4,17 @@ on:
   pull_request:
     branches:
       - "**"
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docs-website/**"
   push:
     branches:
       - master
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docs-website/**"
   # release:
   #   types: [published, edited]
 

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -11,7 +11,7 @@ on:
     branches:
       - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/metadata-ingestion.yml"
       - "metadata-ingestion/**"
       - "metadata-models/**"
   release:

diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -3,9 +3,8 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-models/**"
   release:
     types: [published]
 

diff --git a/build.gradle b/build.gradle
@@ -41,7 +41,7 @@ buildscript {
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id 'com.palantir.docker' version '0.35.0'
+  id 'com.palantir.docker' version '0.35.0' apply false
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }

diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java
@@ -1,6 +1,8 @@
 package client;
 
+import com.linkedin.metadata.config.kafka.ProducerConfiguration;
 import com.typesafe.config.Config;
+import config.ConfigurationProvider;
 import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -35,12 +37,12 @@ public class KafkaTrackingProducer {
     private final KafkaProducer<String, String> _producer;
 
     @Inject
-    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle) {
+    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle, final ConfigurationProvider configurationProvider) {
         _isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled");
 
         if (_isEnabled) {
             _logger.debug("Analytics tracking is enabled");
-            _producer = createKafkaProducer(config);
+            _producer = createKafkaProducer(config, configurationProvider.getKafka().getProducer());
 
             lifecycle.addStopHook(
                     () -> {
@@ -62,13 +64,15 @@ public void send(ProducerRecord<String, String> record) {
         _producer.send(record);
     }
 
-    private static KafkaProducer createKafkaProducer(Config config) {
+    private static KafkaProducer createKafkaProducer(Config config, ProducerConfiguration producerConfiguration) {
         final Properties props = new Properties();
         props.put(ProducerConfig.CLIENT_ID_CONFIG, "datahub-frontend");
         props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.getString("analytics.kafka.delivery.timeout.ms"));
         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getString("analytics.kafka.bootstrap.server"));
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // Actor urn.
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // JSON object.
+        props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, producerConfiguration.getMaxRequestSize());
+        props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerConfiguration.getCompressionType());
 
         final String securityProtocolConfig = "analytics.kafka.security.protocol";
         if (config.hasPath(securityProtocolConfig)

diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java
@@ -1,6 +1,7 @@
 package config;
 
 import com.linkedin.metadata.config.cache.CacheConfiguration;
+import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import lombok.Data;
 
@@ -11,14 +12,17 @@
 
 /**
  * Minimal sharing between metadata-service and frontend
- * Initially for use of client caching configuration.
  * Does not use the factories module to avoid transitive dependencies.
  */
 @EnableConfigurationProperties
 @PropertySource(value = "application.yml", factory = YamlPropertySourceFactory.class)
 @ConfigurationProperties
 @Data
 public class ConfigurationProvider {
+    /**
+     * Kafka related configs.
+     */
+    private KafkaConfiguration kafka;
 
     /**
      * Configuration for caching

diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -275,6 +275,7 @@ export const HomePageHeader = () => {
                         viewsEnabled={viewsEnabled}
                         combineSiblings
                         showQuickFilters
+                        showViewAllResults
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>

diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -190,7 +190,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                             <Input
                                 data-testid="cli-version-input"
                                 className="cli-version-input"
-                                placeholder="(e.g. 0.10.5)"
+                                placeholder="(e.g. 0.12.0)"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
                             />

diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -119,6 +119,7 @@ interface Props {
     setIsSearchBarFocused?: (isSearchBarFocused: boolean) => void;
     onFocus?: () => void;
     onBlur?: () => void;
+    showViewAllResults?: boolean;
 }
 
 const defaultProps = {
@@ -146,6 +147,7 @@ export const SearchBar = ({
     setIsSearchBarFocused,
     onFocus,
     onBlur,
+    showViewAllResults = false,
 }: Props) => {
     const history = useHistory();
     const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
@@ -203,7 +205,7 @@ export const SearchBar = ({
     const { quickFilters, selectedQuickFilter, setSelectedQuickFilter } = useQuickFiltersContext();
 
     const autoCompleteQueryOptions = useMemo(() => {
-        if (effectiveQuery === '') return [];
+        if (effectiveQuery === '' || !showViewAllResults) return [];
 
         return [
             {
@@ -212,7 +214,7 @@ export const SearchBar = ({
                 type: EXACT_AUTOCOMPLETE_OPTION_TYPE,
             },
         ];
-    }, [effectiveQuery]);
+    }, [effectiveQuery, showViewAllResults]);
 
     const autoCompleteEntityOptions = useMemo(() => {
         return suggestions.map((suggestion: AutoCompleteResultForEntity) => {

diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx
@@ -107,6 +107,7 @@ export const SearchHeader = ({
                     combineSiblings
                     fixAutoComplete
                     showQuickFilters
+                    showViewAllResults
                 />
             </LogoSearchContainer>
             <NavGroup>

diff --git a/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts b/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
@@ -3,7 +3,7 @@ import { useLocation, useParams } from 'react-router';
 import { useMemo } from 'react';
 import { FacetFilterInput, EntityType } from '../../types.generated';
 import { useEntityRegistry } from '../useEntityRegistry';
-import { ENTITY_FILTER_NAME, FILTER_DELIMITER, UnionType } from './utils/constants';
+import { ENTITY_FILTER_NAME, UnionType } from './utils/constants';
 import { useUserContext } from '../context/useUserContext';
 import useFilters from './utils/useFilters';
 import { generateOrFilters } from './utils/generateOrFilters';
@@ -27,12 +27,6 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     const sortInput = useSortInput();
 
     const filters: Array<FacetFilterInput> = useFilters(params);
-    const nonNestedFilters = filters.filter(
-        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
-    const nestedFilters = filters.filter(
-        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
     const entityFilters: Array<EntityType> = useMemo(
         () =>
             filters
@@ -43,8 +37,8 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     );
 
     const orFilters = useMemo(
-        () => generateOrFilters(unionType, nonNestedFilters, nestedFilters),
-        [nonNestedFilters, nestedFilters, unionType],
+        () => generateOrFilters(unionType, filters, excludedFilterFields),
+        [filters, excludedFilterFields, unionType],
     );
 
     return { entityFilters, query, unionType, filters, orFilters, viewUrn, page, activeType, sortInput };

diff --git a/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts b/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
@@ -1,7 +1,7 @@
 import {
     DOMAINS_FILTER_NAME,
     ENTITY_SUB_TYPE_FILTER_NAME,
-    ENTITY_TYPE_FILTER_NAME,
+    ENTITY_FILTER_NAME,
     TAGS_FILTER_NAME,
     UnionType,
 } from '../constants';
@@ -10,7 +10,7 @@ import { generateOrFilters } from '../generateOrFilters';
 describe('generateOrFilters', () => {
     it('should generate orFilters with UnionType.AND', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.AND, filters);
@@ -24,7 +24,7 @@ describe('generateOrFilters', () => {
 
     it('should generate orFilters with UnionType.OR', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.OR, filters);
@@ -43,17 +43,23 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.AND, filters, nestedFilters);
+        // const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
+        const orFilters = generateOrFilters(UnionType.AND, filters);
 
         expect(orFilters).toMatchObject([
             {
-                and: [...filters, { field: '_entityType', values: ['CONTAINER'] }],
+                and: [
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+                    { field: '_entityType', values: ['CONTAINER'] },
+                ],
             },
             {
                 and: [
-                    ...filters,
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
                     { field: '_entityType', values: ['DATASET'] },
                     { field: 'typeNames', values: ['table'] },
                 ],
@@ -65,9 +71,9 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.OR, filters, nestedFilters);
+        const orFilters = generateOrFilters(UnionType.OR, filters);
 
         expect(orFilters).toMatchObject([
             {
@@ -87,4 +93,18 @@ describe('generateOrFilters', () => {
             },
         ]);
     });
+
+    it('should generate orFilters and exclude filters with a provided exclude field', () => {
+        const filters = [
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+        ];
+        const orFilters = generateOrFilters(UnionType.AND, filters, [ENTITY_FILTER_NAME]);
+
+        expect(orFilters).toMatchObject([
+            {
+                and: [{ field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] }],
+            },
+        ]);
+    });
 });
diff --git a/datahub-web-react/src/app/search/utils/generateOrFilters.ts b/datahub-web-react/src/app/search/utils/generateOrFilters.ts
@@ -26,28 +26,34 @@ function generateInputWithNestedFilters(filters: FacetFilterInput[], nestedFilte
 export function generateOrFilters(
     unionType: UnionType,
     filters: FacetFilterInput[],
-    nestedFilters: FacetFilterInput[] = [],
+    excludedFilterFields: string[] = [],
 ): AndFilterInput[] {
-    if ((filters?.length || 0) === 0 && nestedFilters.length === 0) {
+    if ((filters?.length || 0) === 0) {
         return [];
     }
+    const nonNestedFilters = filters.filter(
+        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
+    const nestedFilters = filters.filter(
+        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
 
     if (unionType === UnionType.OR) {
         const orFiltersWithNestedFilters = generateInputWithNestedFilters([], nestedFilters);
-        const orFilters = filters.map((filter) => ({
+        const orFilters = nonNestedFilters.map((filter) => ({
             and: [filter],
         }));
         return [...orFilters, ...orFiltersWithNestedFilters];
     }
-    const andFiltersWithNestedFilters = generateInputWithNestedFilters(filters, nestedFilters);
+    const andFiltersWithNestedFilters = generateInputWithNestedFilters(nonNestedFilters, nestedFilters);
 
     if (andFiltersWithNestedFilters.length) {
         return andFiltersWithNestedFilters;
     }
 
     return [
         {
-            and: filters,
+            and: nonNestedFilters,
         },
     ];
 }
diff --git a/docker/broker/env/docker.env b/docker/broker/env/docker.env
@@ -5,4 +5,6 @@ KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9
 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
 KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
-KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+KAFKA_MESSAGE_MAX_BYTES=5242880
+KAFKA_MAX_MESSAGE_BYTES=5242880