Merge branch 'master' into kafka-message-size-parameters

datahub-project · Oct 21, 2023 · 666c3b5 · 666c3b5
2 parents d8d5ecd + 63599c9
commit 666c3b5
Show file tree

Hide file tree

Showing 31 changed files with 223 additions and 68 deletions.
diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx
@@ -41,7 +41,11 @@ export default function UpdateDescriptionModal({ title, description, original, o
             footer={
                 <>
                     <Button onClick={onClose}>Cancel</Button>
-                    <Button onClick={() => onSubmit(updatedDesc)} disabled={updatedDesc === description}>
+                    <Button
+                        onClick={() => onSubmit(updatedDesc)}
+                        disabled={updatedDesc === description}
+                        data-testid="description-modal-update-button"
+                    >
                         Update
                     </Button>
                 </>

diff --git a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx
@@ -57,7 +57,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
 
     return (
         <>
-            <Button icon={<PlusOutlined />} onClick={showModal} {...buttonProps}>
+            <Button data-testid="add-link-button" icon={<PlusOutlined />} onClick={showModal} {...buttonProps}>
                 Add Link
             </Button>
             <Modal
@@ -69,13 +69,14 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                     <Button type="text" onClick={handleClose}>
                         Cancel
                     </Button>,
-                    <Button form="addLinkForm" key="submit" htmlType="submit">
+                    <Button data-testid="add-link-modal-add-button" form="addLinkForm" key="submit" htmlType="submit">
                         Add
                     </Button>,
                 ]}
             >
                 <Form form={form} name="addLinkForm" onFinish={handleAdd} layout="vertical">
                     <Form.Item
+                        data-testid="add-link-modal-url"
                         name="url"
                         label="URL"
                         rules={[
@@ -93,6 +94,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                         <Input placeholder="https://" autoFocus />
                     </Form.Item>
                     <Form.Item
+                        data-testid="add-link-modal-label"
                         name="label"
                         label="Label"
                         rules={[

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx
@@ -60,6 +60,7 @@ export const DocumentationTab = ({ properties }: { properties?: Props }) => {
                     <TabToolbar>
                         <div>
                             <Button
+                                data-testid="edit-documentation-button"
                                 type="text"
                                 onClick={() => routeToTab({ tabName: 'Documentation', tabParams: { editing: true } })}
                             >

diff --git a/...eb-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx b/...eb-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx
@@ -15,7 +15,7 @@ export const DescriptionEditorToolbar = ({ disableSave, onClose, onSave }: Descr
             <Button type="text" onClick={onClose}>
                 Back
             </Button>
-            <Button onClick={onSave} disabled={disableSave}>
+            <Button data-testid="description-editor-save-button" onClick={onSave} disabled={disableSave}>
                 <CheckOutlined /> Save
             </Button>
         </TabToolbar>

diff --git a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
@@ -40,6 +40,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         Cancel
                     </Button>
                     <Button
+                        data-testid="secret-modal-create-button"
                         id="createSecretButton"
                         onClick={() =>
                             onSubmit?.(
@@ -71,6 +72,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         Give your secret a name. This is what you&apos;ll use to reference the secret from your recipes.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-name-input"
                         name={NAME_FIELD_NAME}
                         rules={[
                             {
@@ -91,6 +93,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         The value of your secret, which will be encrypted and stored securely within DataHub.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-value-input"
                         name={VALUE_FIELD_NAME}
                         rules={[
                             {
@@ -110,6 +113,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         An optional description to help keep track of your secret.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-description-input"
                         name={DESCRIPTION_FIELD_NAME}
                         rules={[{ whitespace: true }, { min: 1, max: 500 }]}
                         hasFeedback

diff --git a/datahub-web-react/src/app/ingest/secret/SecretsList.tsx b/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
@@ -176,7 +176,11 @@ export const SecretsList = () => {
             <div>
                 <TabToolbar>
                     <div>
-                        <Button type="text" onClick={() => setIsCreatingSecret(true)}>
+                        <Button
+                            data-testid="create-secret-button"
+                            type="text"
+                            onClick={() => setIsCreatingSecret(true)}
+                        >
                             <PlusOutlined /> Create new secret
                         </Button>
                     </div>

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -358,7 +358,12 @@ export const IngestionSourceList = () => {
             <SourceContainer>
                 <TabToolbar>
                     <div>
-                        <Button id={INGESTION_CREATE_SOURCE_ID} type="text" onClick={() => setIsBuildingSource(true)}>
+                        <Button
+                            id={INGESTION_CREATE_SOURCE_ID}
+                            type="text"
+                            onClick={() => setIsBuildingSource(true)}
+                            data-testid="create-ingestion-source-button"
+                        >
                             <PlusOutlined /> Create new source
                         </Button>
                         <Button id={INGESTION_REFRESH_SOURCES_ID} type="text" onClick={onRefresh}>

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
@@ -119,7 +119,11 @@ export function LastStatusColumn({ status, record, setFocusExecutionUrn }: LastS
     return (
         <StatusContainer>
             {Icon && <Icon style={{ color, fontSize: 14 }} />}
-            <StatusButton type="link" onClick={() => setFocusExecutionUrn(record.lastExecUrn)}>
+            <StatusButton
+                data-testid="ingestion-source-table-status"
+                type="link"
+                onClick={() => setFocusExecutionUrn(record.lastExecUrn)}
+            >
                 <Typography.Text strong style={{ color, marginLeft: 8 }}>
                     {text || 'Pending...'}
                 </Typography.Text>
@@ -159,7 +163,11 @@ export function ActionsColumn({
                 </Tooltip>
             )}
             {!record.cliIngestion && (
-                <Button style={{ marginRight: 16 }} onClick={() => onEdit(record.urn)}>
+                <Button
+                    data-testid="ingestion-source-table-edit-button"
+                    style={{ marginRight: 16 }}
+                    onClick={() => onEdit(record.urn)}
+                >
                     EDIT
                 </Button>
             )}

diff --git a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx
@@ -167,7 +167,11 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
             <ControlsContainer>
                 <Button onClick={prev}>Previous</Button>
                 <div>
-                    <Button disabled={!interval || interval.length === 0 || cronAsText.error} onClick={onClickNext}>
+                    <Button
+                        data-testid="ingestion-schedule-next-button"
+                        disabled={!interval || interval.length === 0 || cronAsText.error}
+                        onClick={onClickNext}
+                    >
                         Next
                     </Button>
                 </div>

diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -123,6 +123,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                 <Button onClick={prev}>Previous</Button>
                 <div>
                     <SaveButton
+                        data-testid="ingestion-source-save-button"
                         disabled={!(state.name !== undefined && state.name.length > 0)}
                         onClick={() => onClickCreate(false)}
                     >

diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx
@@ -86,10 +86,20 @@ function RecipeBuilder(props: Props) {
                     {sourceConfigs?.displayName} Recipe
                 </Title>
                 <ButtonsWrapper>
-                    <StyledButton type="text" isSelected={isViewingForm} onClick={() => switchViews(true)}>
+                    <StyledButton
+                        type="text"
+                        isSelected={isViewingForm}
+                        onClick={() => switchViews(true)}
+                        data-testid="recipe-builder-form-button"
+                    >
                         <FormOutlined /> Form
                     </StyledButton>
-                    <StyledButton type="text" isSelected={!isViewingForm} onClick={() => switchViews(false)}>
+                    <StyledButton
+                        type="text"
+                        isSelected={!isViewingForm}
+                        onClick={() => switchViews(false)}
+                        data-testid="recipe-builder-yaml-button"
+                    >
                         <CodeOutlined /> YAML
                     </StyledButton>
                 </ButtonsWrapper>
@@ -114,7 +124,9 @@ function RecipeBuilder(props: Props) {
                         <Button disabled={isEditing} onClick={goToPrevious}>
                             Previous
                         </Button>
-                        <Button onClick={onClickNext}>Next</Button>
+                        <Button data-testid="recipe-builder-next-button" onClick={onClickNext}>
+                            Next
+                        </Button>
                     </ControlsContainer>
                 </>
             )}

diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
@@ -221,6 +221,7 @@ def report_table_dropped(self, table: str) -> None:
     SourceCapability.DELETION_DETECTION,
     "Enabled by default when stateful ingestion is turned on.",
 )
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class GlueSource(StatefulIngestionSourceBase):
     """
     Note: if you also have files in S3 that you'd like to ingest, we recommend you use Glue's built-in data catalog. See [here](../../../../docs/generated/ingestion/sources/s3.md) for a quick guide on how to set up a crawler on Glue and ingest the outputs with DataHub.

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -153,6 +153,7 @@ def cleanup(config: BigQueryV2Config) -> None:
 )
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
 @capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration")
+@capability(SourceCapability.LINEAGE_FINE, "Optionally enabled via configuration")
 @capability(
     SourceCapability.USAGE_STATS,
     "Enabled by default, can be disabled via configuration `include_usage_statistics`",

diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py
@@ -1096,6 +1096,7 @@ def transform_connector_config(
 @config_class(KafkaConnectSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class KafkaConnectSource(StatefulIngestionSourceBase):
     config: KafkaConnectSourceConfig
     report: KafkaConnectSourceReport

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -103,6 +103,11 @@
 @capability(
     SourceCapability.OWNERSHIP, "Enabled by default, configured using `extract_owners`"
 )
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default, configured using `extract_column_level_lineage`",
+)
 @capability(
     SourceCapability.USAGE_STATS,
     "Enabled by default, configured using `extract_usage_history`",
@@ -1128,7 +1133,6 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
     def emit_independent_looks_mcp(
         self, dashboard_element: LookerDashboardElement
     ) -> Iterable[MetadataWorkUnit]:
-
         yield from auto_workunit(
             stream=self._make_chart_metadata_events(
                 dashboard_element=dashboard_element,

diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py
@@ -80,6 +80,7 @@ def remove_trailing_slash(cls, v):
 @config_class(MetabaseConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
 class MetabaseSource(Source):
     """
     This plugin extracts Charts, dashboards, and associated metadata. This plugin is in beta and has only been tested

diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
@@ -23,11 +23,17 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    Source,
+    SourceCapability,
+    SourceReport,
+)
 from datahub.ingestion.api.source_helpers import (
     auto_status_aspect,
     auto_workunit_reporter,
@@ -121,6 +127,8 @@ def version_must_be_1(cls, v):
 @platform_name("File Based Lineage")
 @config_class(LineageFileSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.LINEAGE_COARSE, "Specified in the lineage file.")
+@capability(SourceCapability.LINEAGE_FINE, "Specified in the lineage file.")
 @dataclass
 class LineageFileSource(Source):
     """

diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py
@@ -98,6 +98,7 @@ class HTTPError429(HTTPError):
 @config_class(ModeConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
 class ModeSource(Source):
     """
 

diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -26,11 +26,12 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceCapability, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.schema_classes import (
     DataFlowInfoClass,
@@ -360,6 +361,7 @@ def report_dropped(self, ent_name: str) -> None:
 @platform_name("NiFi", id="nifi")
 @config_class(NifiSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.LINEAGE_COARSE, "Supported. See docs for limitations")
 class NifiSource(Source):
     """
     This plugin extracts the following:

diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -264,7 +264,6 @@ def extract_lineage(
                 )
 
         if len(upstream) > 0:
-
             upstream_lineage_class: UpstreamLineageClass = UpstreamLineageClass(
                 upstreams=upstream,
                 fineGrainedLineages=cll_lineage or None,
@@ -1139,6 +1138,10 @@ def report_to_datahub_work_units(
     SourceCapability.OWNERSHIP,
     "Disabled by default, configured using `extract_ownership`",
 )
+@capability(
+    SourceCapability.LINEAGE_COARSE,
+    "Enabled by default, configured using `extract_lineage`.",
+)
 @capability(
     SourceCapability.LINEAGE_FINE,
     "Disabled by default, configured using `extract_column_level_lineage`. ",

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py
@@ -20,11 +20,17 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    Source,
+    SourceCapability,
+    SourceReport,
+)
 from datahub.ingestion.api.source_helpers import auto_workunit_reporter
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.graph.client import DataHubGraph
@@ -83,6 +89,8 @@ def compute_stats(self) -> None:
 @platform_name("SQL Queries")
 @config_class(SqlQueriesSourceConfig)
 @support_status(SupportStatus.TESTING)
+@capability(SourceCapability.LINEAGE_COARSE, "Parsed from SQL queries")
+@capability(SourceCapability.LINEAGE_FINE, "Parsed from SQL queries")
 class SqlQueriesSource(Source):
     # TODO: Documentation
     urns: Optional[Set[str]]

diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py
@@ -142,6 +142,7 @@ def get_filter_name(filter_obj):
 @capability(
     SourceCapability.DELETION_DETECTION, "Optionally enabled via stateful_ingestion"
 )
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
 class SupersetSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:

diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
@@ -452,6 +452,10 @@ class TableauSourceReport(StaleEntityRemovalSourceReport):
 @capability(SourceCapability.OWNERSHIP, "Requires recipe configuration")
 @capability(SourceCapability.TAGS, "Requires recipe configuration")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default, configure using `extract_column_level_lineage`",
+)
 class TableauSource(StatefulIngestionSourceBase):
     platform = "tableau"
 
@@ -533,7 +537,7 @@ def fetch_projects():
                     path=[],
                 )
             # Set parent project name
-            for project_id, project in all_project_map.items():
+            for _project_id, project in all_project_map.items():
                 if (
                     project.parent_id is not None
                     and project.parent_id in all_project_map

diff --git a/...rc/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/...rc/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
@@ -133,7 +133,7 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
     } else if (fieldType == FieldType.DATETIME) {
       mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
     } else if (fieldType == FieldType.OBJECT) {
-      mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
+      mappingForField.put(TYPE, ESUtils.OBJECT_FIELD_TYPE);
     } else {
       log.info("FieldType {} has no mappings implemented", fieldType);
     }