Merge branch 'master' into CUS-903-dbt-meta-mapping

datahub-project · Oct 3, 2023 · cecf678 · cecf678
2 parents c27b5d6 + 0a5e7d1
commit cecf678
Show file tree

Hide file tree

Showing 30 changed files with 464 additions and 124 deletions.
diff --git a/build.gradle b/build.gradle
@@ -200,8 +200,8 @@ project.ext.externalDependency = [
     'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
     'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11',
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
-    'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12',
-    'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41',
+    'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
+    'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
     'testngJava8': 'org.testng:testng:7.5.1',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,

diff --git a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java
@@ -6,6 +6,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+import com.google.common.collect.ImmutableSet;
 import com.linkedin.metadata.models.registry.config.Entities;
 import com.linkedin.metadata.models.registry.config.Entity;
 import org.gradle.internal.Pair;
@@ -16,7 +17,12 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.Spliterators;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -37,10 +43,23 @@ public class OpenApiEntities {
     private String entityRegistryYaml;
     private Path combinedDirectory;
 
-    private final static Set<String> SUPPORTED_ASPECT_PATHS = Set.of(
-            "domains", "ownership", "deprecation", "status", "globalTags", "glossaryTerms", "dataContractInfo",
-            "browsePathsV2"
-    );
+    private final static ImmutableSet<Object> SUPPORTED_ASPECT_PATHS = ImmutableSet.builder()
+                .add("domains")
+                .add("ownership")
+                .add("deprecation")
+                .add("status")
+                .add("globalTags")
+                .add("glossaryTerms")
+                .add("dataContractInfo")
+                .add("browsePathsV2")
+                .add("datasetProperties").add("editableDatasetProperties")
+                .add("chartInfo").add("editableChartProperties")
+                .add("dashboardInfo").add("editableDashboardProperties")
+                .add("notebookInfo").add("editableNotebookProperties")
+                .add("dataProductProperties")
+                .add("institutionalMemory")
+                .build();
+
 
     public OpenApiEntities(JsonNodeFactory NODE_FACTORY) {
         this.NODE_FACTORY = NODE_FACTORY;

diff --git a/...b-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/...b-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java
@@ -17,6 +17,7 @@ public class EntityTypeMapper {
       ImmutableMap.<EntityType, String>builder()
           .put(EntityType.DATASET, "dataset")
           .put(EntityType.ROLE, "role")
+          .put(EntityType.ASSERTION, Constants.ASSERTION_ENTITY_NAME)
           .put(EntityType.CORP_USER, "corpuser")
           .put(EntityType.CORP_GROUP, "corpGroup")
           .put(EntityType.DATA_PLATFORM, "dataPlatform")
@@ -25,6 +26,7 @@ public class EntityTypeMapper {
           .put(EntityType.TAG, "tag")
           .put(EntityType.DATA_FLOW, "dataFlow")
           .put(EntityType.DATA_JOB, "dataJob")
+          .put(EntityType.DATA_PROCESS_INSTANCE, Constants.DATA_PROCESS_INSTANCE_ENTITY_NAME)
           .put(EntityType.GLOSSARY_TERM, "glossaryTerm")
           .put(EntityType.GLOSSARY_NODE, "glossaryNode")
           .put(EntityType.MLMODEL, "mlModel")

diff --git a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx
@@ -51,7 +51,7 @@ export class DataProductEntity implements Entity<DataProduct> {
 
     isSearchEnabled = () => true;
 
-    isBrowseEnabled = () => false;
+    isBrowseEnabled = () => true;
 
     isLineageEnabled = () => false;
 

diff --git a/datahub-web-react/src/app/entity/shared/components/styled/DemoButton.tsx b/datahub-web-react/src/app/entity/shared/components/styled/DemoButton.tsx
@@ -12,7 +12,7 @@ export default function DemoButton() {
     return (
         <StyledButton
             type="primary"
-            href="https://www.acryldata.io/datahub-sign-up"
+            href="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup"
             target="_blank"
             rel="noopener noreferrer"
         >

diff --git a/datahub-web-react/src/app/entity/shared/embed/EmbeddedProfile.tsx b/datahub-web-react/src/app/entity/shared/embed/EmbeddedProfile.tsx
@@ -55,6 +55,8 @@ export default function EmbeddedProfile<T>({ urn, entityType, getOverridePropert
         return <NonExistentEntityPage />;
     }
 
+    const readOnly = false;
+
     return (
         <EntityContext.Provider
             value={{
@@ -80,15 +82,15 @@ export default function EmbeddedProfile<T>({ urn, entityType, getOverridePropert
                     <StyledDivider />
                     <UpstreamHealth />
                     <StyledDivider />
-                    <SidebarAboutSection readOnly />
+                    <SidebarAboutSection readOnly={readOnly} />
                     <StyledDivider />
-                    <SidebarOwnerSection readOnly />
+                    <SidebarOwnerSection readOnly={readOnly} />
                     <StyledDivider />
-                    <SidebarTagsSection readOnly properties={{ hasTags: true, hasTerms: true }} />
+                    <SidebarTagsSection readOnly={readOnly} properties={{ hasTags: true, hasTerms: true }} />
                     <StyledDivider />
-                    <SidebarDomainSection readOnly />
+                    <SidebarDomainSection readOnly={readOnly} />
                     <StyledDivider />
-                    <DataProductSection readOnly />
+                    <DataProductSection readOnly={readOnly} />
                 </>
             )}
         </EntityContext.Provider>

diff --git a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx
@@ -46,7 +46,7 @@ export default function AcrylDemoBanner() {
                 <TextContent>
                     DataHub is already the industry&apos;s #1 Open Source Data Catalog.{' '}
                     <StyledLink
-                        href="https://www.acryldata.io/datahub-sign-up"
+                        href="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup"
                         target="_blank"
                         rel="noopener noreferrer"
                     >

diff --git a/datahub-web-react/src/app/home/HomePageRecommendations.tsx b/datahub-web-react/src/app/home/HomePageRecommendations.tsx
@@ -95,6 +95,7 @@ const simpleViewEntityTypes = [
     EntityType.Dashboard,
     EntityType.GlossaryNode,
     EntityType.GlossaryTerm,
+    EntityType.DataProduct,
 ];
 
 export const HomePageRecommendations = ({ user }: Props) => {

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
@@ -23,7 +23,7 @@ module.exports = {
       announcementBar: {
         id: "announcement",
         content:
-          '<div><img src="/img/acryl-logo-white-mark.svg" /><p><strong>Managed DataHub</strong><span> &nbsp;Acryl Data delivers an easy to consume DataHub platform for the enterprise</span></p></div> <a href="https://www.acryldata.io/datahub-sign-up" target="_blank" class="button button--primary">Sign up for Managed DataHub&nbsp;→</a>',
+          '<div><img src="/img/acryl-logo-white-mark.svg" /><p><strong>Managed DataHub</strong><span> &nbsp;Acryl Data delivers an easy to consume DataHub platform for the enterprise</span></p></div> <a href="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup" target="_blank" class="button button--primary">Sign up for Managed DataHub&nbsp;→</a>',
         backgroundColor: "#070707",
         textColor: "#ffffff",
         isCloseable: false,

diff --git a/docs-website/src/pages/_components/CardCTAs/index.js b/docs-website/src/pages/_components/CardCTAs/index.js
@@ -8,17 +8,17 @@ const cardsContent = [
   {
     label: "Data Mesh",
     title: "Data Products, Delivered",
-    url: "https://www.acryldata.io/blog/data-products-in-datahub-everything-you-need-to-know",
+    url: "https://www.acryldata.io/blog/data-products-in-datahub-everything-you-need-to-know?utm_source=datahub&utm_medium=referral&utm_content=blog",
   },
   {
     label: "Data Contracts",
     title: "End-to-end Reliability in Data",
-    url: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management",
+    url: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog",
   },
   {
     label: "Shift Left",
     title: "Developer-friendly Data Governance",
-    url: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2",
+    url: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2?utm_source=datahub&utm_medium=referral&utm_content=blog",
   },
 ];
 

diff --git a/docs-website/src/pages/_components/Section/index.js b/docs-website/src/pages/_components/Section/index.js
@@ -18,7 +18,7 @@ const PromoSection = () => (
       <img src={useBaseUrl("/img/acryl-logo-white-mark.svg")} />
       <h2>Managed DataHub</h2>
       <p>Acryl Data delivers an easy to consume DataHub platform for the enterprise</p>
-      <a href="https://www.acryldata.io/datahub-beta" target="_blank" className="button button--primary button--lg">
+      <a href="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup" target="_blank" className="button button--primary button--lg">
         Sign up for Managed DataHub →
       </a>
     </div>

diff --git a/docs/act-on-metadata/impact-analysis.md b/docs/act-on-metadata/impact-analysis.md
@@ -38,7 +38,7 @@ Follow these simple steps to understand the full dependency chain of your data e
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/impact-analysis-filter-dependencies.png"/>
 </p>
 
-4. Slice and dice the result list by Entity Type, Platfrom, Owner, and more to isolate the relevant dependencies
+4. Slice and dice the result list by Entity Type, Platform, Owner, and more to isolate the relevant dependencies
 
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/impact-analysis-apply-filters.png"/>

diff --git a/docs/ownership/ownership-types.md b/docs/ownership/ownership-types.md
@@ -7,7 +7,7 @@ import TabItem from '@theme/TabItem';
 <FeatureAvailability/>
 
 **🤝 Version compatibility**
-> Open Source DataHub: **0.10.3** | Acryl: **0.2.8**
+> Open Source DataHub: **0.10.4** | Acryl: **0.2.8**
 
 ## What are Custom Ownership Types?
 Custom Ownership Types are an improvement on the way to establish ownership relationships between users and the data assets they manage within DataHub.

diff --git a/docs/saas.md b/docs/saas.md
@@ -5,10 +5,10 @@ Sign up for fully managed, hassle-free and secure SaaS service for DataHub, prov
 <p>
 <a
     className="button button--primary button--lg"
-    href="https://www.acryldata.io/datahub-beta" 
+    href="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup"
     target="_blank" >
     Sign up
 </a>
 </p>
 
-Refer to [Managed Datahub Exclusives](/docs/managed-datahub/managed-datahub-overview.md) for more information. 
+Refer to [Managed Datahub Exclusives](/docs/managed-datahub/managed-datahub-overview.md) for more information.
diff --git a/metadata-ingestion/examples/ownership/ownership_type.json b/metadata-ingestion/examples/ownership/ownership_type.json
@@ -1,7 +1,14 @@
-{
-  "urn": "urn:li:ownershipType:architect",
-  "info": {
-    "name": "Architect",
-    "description": "Technical person responsible for the asset"
+[
+  {
+    "auditHeader":null,
+    "entityType":"ownershipType",
+    "entityUrn": "urn:li:ownershipType:architect",
+    "changeType":"UPSERT",
+    "aspectName":"ownershipTypeInfo",
+    "aspect":{
+      "value":"{\"name\": \"Architect\", \"description\": \"Technical person responsible for the asset\", \"created\": {\"time\": 1674291843000,  \"actor\": \"urn:li:corpuser:jdoe\",  \"impersonator\": null},\n\"lastModified\": {\"time\": 1674291843000,  \"actor\": \"urn:li:corpuser:jdoe\",  \"impersonator\": null}}",
+      "contentType":"application/json"
+    },
+    "systemMetadata":null
   }
-}
+]
diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py
@@ -805,7 +805,7 @@ def get_related_entities(
                 url=relationship_endpoint,
                 params={
                     "urn": entity_urn,
-                    "direction": direction,
+                    "direction": direction.value,
                     "relationshipTypes": relationship_types,
                     "start": start,
                 },

diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
@@ -351,3 +351,5 @@ cache:
           status: 20
           corpUserCredentials: 20
           corpUserSettings: 20
+
+springdoc.api-docs.groups.enabled: true
diff --git a/metadata-service/health-servlet/src/main/java/com/datahub/health/config/SpringWebConfig.java b/metadata-service/health-servlet/src/main/java/com/datahub/health/config/SpringWebConfig.java