Merge branch 'master' into feat/policyFixes

datahub-project · Nov 3, 2023 · 395a1bd · 395a1bd
2 parents 9339ce3 + 148ad1a
commit 395a1bd
Show file tree

Hide file tree

Showing 260 changed files with 14,400 additions and 7,723 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -12,7 +12,7 @@ on:
     branches:
       - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/airflow-plugin.yml"
       - "metadata-ingestion-modules/airflow-plugin/**"
       - "metadata-ingestion/**"
       - "metadata-models/**"

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -4,17 +4,13 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docker/**"
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-integration"
   pull_request:
     branches:
       - "**"
-    paths-ignore:
-      - "docker/**"
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-integration"
   release:
     types: [published]
 

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -851,8 +851,14 @@ jobs:
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
-          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
+          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -4,9 +4,17 @@ on:
   pull_request:
     branches:
       - "**"
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docs-website/**"
   push:
     branches:
       - master
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docs-website/**"
   # release:
   #   types: [published, edited]
 

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -11,7 +11,7 @@ on:
     branches:
       - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/metadata-ingestion.yml"
       - "metadata-ingestion/**"
       - "metadata-models/**"
   release:

diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -3,9 +3,8 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-models/**"
   release:
     types: [published]
 

diff --git a/build.gradle b/build.gradle
@@ -39,7 +39,7 @@ buildscript {
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id 'com.palantir.docker' version '0.35.0'
+  id 'com.palantir.docker' version '0.35.0' apply false
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }

diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java
@@ -1,6 +1,8 @@
 package client;
 
+import com.linkedin.metadata.config.kafka.ProducerConfiguration;
 import com.typesafe.config.Config;
+import config.ConfigurationProvider;
 import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -35,12 +37,12 @@ public class KafkaTrackingProducer {
     private final KafkaProducer<String, String> _producer;
 
     @Inject
-    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle) {
+    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle, final ConfigurationProvider configurationProvider) {
         _isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled");
 
         if (_isEnabled) {
             _logger.debug("Analytics tracking is enabled");
-            _producer = createKafkaProducer(config);
+            _producer = createKafkaProducer(config, configurationProvider.getKafka().getProducer());
 
             lifecycle.addStopHook(
                     () -> {
@@ -62,13 +64,15 @@ public void send(ProducerRecord<String, String> record) {
         _producer.send(record);
     }
 
-    private static KafkaProducer createKafkaProducer(Config config) {
+    private static KafkaProducer createKafkaProducer(Config config, ProducerConfiguration producerConfiguration) {
         final Properties props = new Properties();
         props.put(ProducerConfig.CLIENT_ID_CONFIG, "datahub-frontend");
         props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.getString("analytics.kafka.delivery.timeout.ms"));
         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getString("analytics.kafka.bootstrap.server"));
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // Actor urn.
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // JSON object.
+        props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, producerConfiguration.getMaxRequestSize());
+        props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerConfiguration.getCompressionType());
 
         final String securityProtocolConfig = "analytics.kafka.security.protocol";
         if (config.hasPath(securityProtocolConfig)

diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java
@@ -1,6 +1,7 @@
 package config;
 
 import com.linkedin.metadata.config.cache.CacheConfiguration;
+import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import lombok.Data;
 
@@ -11,14 +12,17 @@
 
 /**
  * Minimal sharing between metadata-service and frontend
- * Initially for use of client caching configuration.
  * Does not use the factories module to avoid transitive dependencies.
  */
 @EnableConfigurationProperties
 @PropertySource(value = "application.yml", factory = YamlPropertySourceFactory.class)
 @ConfigurationProperties
 @Data
 public class ConfigurationProvider {
+    /**
+     * Kafka related configs.
+     */
+    private KafkaConfiguration kafka;
 
     /**
      * Configuration for caching

diff --git a/...e/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java b/...e/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java
@@ -5,6 +5,7 @@
 import com.linkedin.datahub.graphql.generated.IngestionConfig;
 import com.linkedin.datahub.graphql.generated.IngestionSchedule;
 import com.linkedin.datahub.graphql.generated.IngestionSource;
+import com.linkedin.datahub.graphql.generated.StringMapEntry;
 import com.linkedin.datahub.graphql.generated.StructuredReport;
 import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper;
 import com.linkedin.entity.EntityResponse;
@@ -21,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -143,6 +145,14 @@ public static IngestionConfig mapIngestionSourceConfig(final DataHubIngestionSou
     result.setVersion(config.getVersion());
     result.setExecutorId(config.getExecutorId());
     result.setDebugMode(config.isDebugMode());
+    if (config.getExtraArgs() != null) {
+      List<StringMapEntry> extraArgs = config.getExtraArgs()
+          .keySet()
+          .stream()
+          .map(key -> new StringMapEntry(key, config.getExtraArgs().get(key)))
+          .collect(Collectors.toList());
+      result.setExtraArgs(extraArgs);
+    }
     return result;
   }
 

diff --git a/...n/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java b/...n/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java
@@ -117,6 +117,9 @@ public CompletableFuture<String> get(final DataFetchingEnvironment environment)
           if (ingestionSourceInfo.getConfig().hasDebugMode()) {
             debugMode = ingestionSourceInfo.getConfig().isDebugMode() ? "true" : "false";
           }
+          if (ingestionSourceInfo.getConfig().hasExtraArgs()) {
+            arguments.putAll(ingestionSourceInfo.getConfig().getExtraArgs());
+          }
           arguments.put(DEBUG_MODE_ARG_NAME, debugMode);
           execInput.setArgs(new StringMap(arguments));
 

diff --git a/...a/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java b/...a/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java
@@ -1,10 +1,12 @@
 package com.linkedin.datahub.graphql.resolvers.ingest.source;
 
 import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.StringMap;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode;
 import com.linkedin.datahub.graphql.exception.DataHubGraphQLException;
+import com.linkedin.datahub.graphql.generated.StringMapEntryInput;
 import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceConfigInput;
 import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceInput;
 import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceScheduleInput;
@@ -17,6 +19,8 @@
 import com.linkedin.mxe.MetadataChangeProposal;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.Map;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 
 import java.net.URISyntaxException;
@@ -108,6 +112,12 @@ private DataHubIngestionSourceConfig mapConfig(final UpdateIngestionSourceConfig
     if (input.getDebugMode() != null) {
       result.setDebugMode(input.getDebugMode());
     }
+    if (input.getExtraArgs() != null) {
+      Map<String, String> extraArgs = input.getExtraArgs()
+          .stream()
+          .collect(Collectors.toMap(StringMapEntryInput::getKey, StringMapEntryInput::getValue));
+      result.setExtraArgs(new StringMap(extraArgs));
+    }
     return result;
   }
 

diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql
@@ -332,6 +332,11 @@ type IngestionConfig {
   Advanced: Whether or not to run ingestion in debug mode
   """
   debugMode: Boolean
+
+  """
+  Advanced: Extra arguments for the ingestion run.
+  """
+  extraArgs: [StringMapEntry!]
 }
 
 """
@@ -483,6 +488,11 @@ input UpdateIngestionSourceConfigInput {
   Whether or not to run ingestion in debug mode
   """
   debugMode: Boolean
+
+  """
+  Extra arguments for the ingestion run.
+  """
+  extraArgs: [StringMapEntryInput!]
 }
 
 """

diff --git a/...m/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java b/...m/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java
@@ -26,7 +26,7 @@ public class UpsertIngestionSourceResolverTest {
       "Test source",
       "mysql", "Test source description",
       new UpdateIngestionSourceScheduleInput("* * * * *", "UTC"),
-      new UpdateIngestionSourceConfigInput("my test recipe", "0.8.18", "executor id", false)
+      new UpdateIngestionSourceConfigInput("my test recipe", "0.8.18", "executor id", false, null)
   );
 
   @Test

diff --git a/datahub-web-react/src/app/entity/group/GroupProfile.tsx b/datahub-web-react/src/app/entity/group/GroupProfile.tsx
@@ -16,7 +16,7 @@ import NonExistentEntityPage from '../shared/entity/NonExistentEntityPage';
 const messageStyle = { marginTop: '10%' };
 
 export enum TabType {
-    Assets = 'Assets',
+    Assets = 'Owner Of',
     Members = 'Members',
 }
 

diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx
@@ -41,7 +41,11 @@ export default function UpdateDescriptionModal({ title, description, original, o
             footer={
                 <>
                     <Button onClick={onClose}>Cancel</Button>
-                    <Button onClick={() => onSubmit(updatedDesc)} disabled={updatedDesc === description}>
+                    <Button
+                        onClick={() => onSubmit(updatedDesc)}
+                        disabled={updatedDesc === description}
+                        data-testid="description-modal-update-button"
+                    >
                         Update
                     </Button>
                 </>

diff --git a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx
@@ -57,7 +57,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
 
     return (
         <>
-            <Button icon={<PlusOutlined />} onClick={showModal} {...buttonProps}>
+            <Button data-testid="add-link-button" icon={<PlusOutlined />} onClick={showModal} {...buttonProps}>
                 Add Link
             </Button>
             <Modal
@@ -69,13 +69,14 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                     <Button type="text" onClick={handleClose}>
                         Cancel
                     </Button>,
-                    <Button form="addLinkForm" key="submit" htmlType="submit">
+                    <Button data-testid="add-link-modal-add-button" form="addLinkForm" key="submit" htmlType="submit">
                         Add
                     </Button>,
                 ]}
             >
                 <Form form={form} name="addLinkForm" onFinish={handleAdd} layout="vertical">
                     <Form.Item
+                        data-testid="add-link-modal-url"
                         name="url"
                         label="URL"
                         rules={[
@@ -93,6 +94,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                         <Input placeholder="https://" autoFocus />
                     </Form.Item>
                     <Form.Item
+                        data-testid="add-link-modal-label"
                         name="label"
                         label="Label"
                         rules={[

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx
@@ -60,6 +60,7 @@ export const DocumentationTab = ({ properties }: { properties?: Props }) => {
                     <TabToolbar>
                         <div>
                             <Button
+                                data-testid="edit-documentation-button"
                                 type="text"
                                 onClick={() => routeToTab({ tabName: 'Documentation', tabParams: { editing: true } })}
                             >

diff --git a/...eb-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx b/...eb-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx
@@ -15,7 +15,7 @@ export const DescriptionEditorToolbar = ({ disableSave, onClose, onSave }: Descr
             <Button type="text" onClick={onClose}>
                 Back
             </Button>
-            <Button onClick={onSave} disabled={disableSave}>
+            <Button data-testid="description-editor-save-button" onClick={onSave} disabled={disableSave}>
                 <CheckOutlined /> Save
             </Button>
         </TabToolbar>

diff --git a/datahub-web-react/src/app/entity/user/UserProfile.tsx b/datahub-web-react/src/app/entity/user/UserProfile.tsx
@@ -17,7 +17,7 @@ export interface Props {
 }
 
 export enum TabType {
-    Assets = 'Assets',
+    Assets = 'Owner Of',
     Groups = 'Groups',
 }
 const ENABLED_TAB_TYPES = [TabType.Assets, TabType.Groups];

diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -275,6 +275,7 @@ export const HomePageHeader = () => {
                         viewsEnabled={viewsEnabled}
                         combineSiblings
                         showQuickFilters
+                        showViewAllResults
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>

diff --git a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
@@ -40,6 +40,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         Cancel
                     </Button>
                     <Button
+                        data-testid="secret-modal-create-button"
                         id="createSecretButton"
                         onClick={() =>
                             onSubmit?.(
@@ -71,6 +72,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         Give your secret a name. This is what you&apos;ll use to reference the secret from your recipes.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-name-input"
                         name={NAME_FIELD_NAME}
                         rules={[
                             {
@@ -91,6 +93,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         The value of your secret, which will be encrypted and stored securely within DataHub.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-value-input"
                         name={VALUE_FIELD_NAME}
                         rules={[
                             {
@@ -110,6 +113,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         An optional description to help keep track of your secret.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-description-input"
                         name={DESCRIPTION_FIELD_NAME}
                         rules={[{ whitespace: true }, { min: 1, max: 500 }]}
                         hasFeedback