Merge branch 'master' into ab-split-ingest-page-view

README.md

@@ -28,7 +28,7 @@ HOSTED_DOCS_ONLY-->
 [![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
 [![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
 [![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
-[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=docs&utm_medium=docs&utm_campaign=docs_page_link)
+[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)
 [![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE)
@@ -106,7 +106,7 @@ We welcome contributions from the community. Please refer to our [Contributing G
 
 ## Community
 
-Join our [Slack workspace](https://datahubproject.io/slack?utm_source=docs&utm_medium=docs&utm_campaign=docs_page_link) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
+Join our [Slack workspace](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
 
 ## Adoption
 

build.gradle

@@ -272,7 +272,7 @@ project.ext.externalDependency = [
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
     'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0',
-    'classGraph': 'io.github.classgraph:classgraph:4.8.168',
+    'classGraph': 'io.github.classgraph:classgraph:4.8.172',
 ]
 
 allprojects {

datahub-frontend/app/auth/AuthModule.java

@@ -62,6 +62,7 @@ public class AuthModule extends AbstractModule {
   private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
   private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
   private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
   private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";
 
   private final com.typesafe.config.Config _configs;
@@ -201,11 +202,13 @@ protected ConfigurationProvider provideConfigurationProvider() {
   protected SystemEntityClient provideEntityClient(
       @Named("systemOperationContext") final OperationContext systemOperationContext,
       final ConfigurationProvider configurationProvider) {
+
     return new SystemRestliEntityClient(
         buildRestliClient(),
         new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
         _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
-        configurationProvider.getCache().getClient().getEntityClient());
+        configurationProvider.getCache().getClient().getEntityClient(),
+        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)));
   }
 
   @Provides

datahub-frontend/conf/application.conf

@@ -288,4 +288,6 @@ systemClientSecret=${?DATAHUB_SYSTEM_CLIENT_SECRET}
 entityClient.retryInterval = 2
 entityClient.retryInterval = ${?ENTITY_CLIENT_RETRY_INTERVAL}
 entityClient.numRetries = 3
-entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
+entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
+entityClient.restli.get.batchSize = 100
+entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java

@@ -14,7 +14,7 @@
 import com.linkedin.datahub.graphql.generated.SearchSuggestion;
 import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
-import com.linkedin.metadata.entity.validation.ValidationUtils;
+import com.linkedin.metadata.entity.validation.ValidationApiUtils;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.utils.SearchUtils;
 import java.net.URISyntaxException;
@@ -89,7 +89,7 @@ public static List<MatchedField> getMatchedFieldEntry(
               if (SearchUtils.isUrn(field.getValue())) {
                 try {
                   Urn urn = Urn.createFromString(field.getValue());
-                  ValidationUtils.validateUrn(
+                  ValidationApiUtils.validateUrn(
                       context.getOperationContext().getEntityRegistry(), urn);
                   matchedField.setEntity(UrnToEntityMapper.map(context, urn));
                 } catch (IllegalArgumentException | URISyntaxException e) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java

@@ -60,6 +60,7 @@ private void mapStructuredPropertyDefinition(
     definition.setQualifiedName(gmsDefinition.getQualifiedName());
     definition.setCardinality(
         PropertyCardinality.valueOf(gmsDefinition.getCardinality().toString()));
+    definition.setImmutable(gmsDefinition.isImmutable());
     definition.setValueType(createDataTypeEntity(gmsDefinition.getValueType()));
     if (gmsDefinition.hasDisplayName()) {
       definition.setDisplayName(gmsDefinition.getDisplayName());

datahub-graphql-core/src/main/resources/entity.graphql

@@ -2685,6 +2685,11 @@ enum FabricType {
     Designates corporation fabrics
     """
     CORP
+
+    """
+    Designates review fabrics
+    """
+    RVW
 }
 
 """

datahub-graphql-core/src/main/resources/properties.graphql

@@ -75,6 +75,11 @@ type StructuredPropertyDefinition {
     Entity types that this structured property can be applied to
     """
     entityTypes: [EntityTypeEntity!]!
+
+    """
+    Whether or not this structured property is immutable
+    """
+    immutable: Boolean!
 }
 
 """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mappers/MapperUtilsTest.java

@@ -9,7 +9,7 @@
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.MatchedField;
-import com.linkedin.metadata.entity.validation.ValidationUtils;
+import com.linkedin.metadata.entity.validation.ValidationApiUtils;
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.snapshot.Snapshot;
@@ -42,7 +42,7 @@ public void testMatchedFieldValidation() throws URISyntaxException {
             "urn:li:dataset:%28urn:li:dataPlatform:s3%2Ctest-datalake-concepts/prog_maintenance%2CPROD%29");
     assertThrows(
         IllegalArgumentException.class,
-        () -> ValidationUtils.validateUrn(entityRegistry, invalidUrn));
+        () -> ValidationApiUtils.validateUrn(entityRegistry, invalidUrn));
 
     QueryContext mockContext = mock(QueryContext.class);
     when(mockContext.getOperationContext())

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java

@@ -125,14 +125,6 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
                 }
               });
 
-      entityService
-          .streamRestoreIndices(opContext, args, x -> context.report().addLine((String) x))
-          .forEach(
-              result -> {
-                context.report().addLine("Rows migrated: " + result.rowsMigrated);
-                context.report().addLine("Rows ignored: " + result.ignored);
-              });
-
       BootstrapStep.setUpgradeResult(opContext, getUpgradeIdUrn(), entityService);
       context.report().addLine("State updated: " + getUpgradeIdUrn());
 

datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditColumn.tsx

@@ -10,7 +10,7 @@ interface Props {
 export function EditColumn({ propertyRow }: Props) {
     const [isEditModalVisible, setIsEditModalVisible] = useState(false);
 
-    if (!propertyRow.structuredProperty) {
+    if (!propertyRow.structuredProperty || propertyRow.structuredProperty?.definition.immutable) {
         return null;
     }
 

datahub-web-react/src/graphql/fragments.graphql

@@ -1245,6 +1245,7 @@ fragment structuredPropertyFields on StructuredPropertyEntity {
         qualifiedName
         description
         cardinality
+        immutable
         valueType {
             info {
                 type

docker/docker-compose.dev.yml

@@ -51,6 +51,7 @@ services:
     volumes:
     - ./datahub-gms/start.sh:/datahub/datahub-gms/scripts/start.sh
     - ./datahub-gms/jetty.xml:/datahub/datahub-gms/scripts/jetty.xml
+    - ./datahub-gms/jetty-jmx.xml:/datahub/datahub-gms/scripts/jetty-jmx.xml
     - ./monitoring/client-prometheus-config.yaml:/datahub/datahub-gms/scripts/prometheus-config.yaml
     - ../metadata-models/src/main/resources/:/datahub/datahub-gms/resources
     - ../metadata-service/war/build/libs/:/datahub/datahub-gms/bin

docker/profiles/docker-compose.gms.yml

@@ -130,6 +130,7 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev
   volumes:
     - ./datahub-gms/start.sh:/datahub/datahub-gms/scripts/start.sh
     - ./datahub-gms/jetty.xml:/datahub/datahub-gms/scripts/jetty.xml
+    - ./datahub-gms/jetty-jmx.xml:/datahub/datahub-gms/scripts/jetty-jmx.xml
     - ./monitoring/client-prometheus-config.yaml:/datahub/datahub-gms/scripts/prometheus-config.yaml
     - ../../metadata-models/src/main/resources/:/datahub/datahub-gms/resources
     - ../../metadata-service/war/build/libs/:/datahub/datahub-gms/bin
@@ -423,4 +424,4 @@ services:
       - debug-consumers
     depends_on:
       datahub-gms-debug-consumers:
-        condition: service_healthy
+        condition: service_healthy

docs-website/sidebars.js

@@ -43,70 +43,138 @@ module.exports = {
         description: "Learn about the features of DataHub.",
       },
       items: [
-        "docs/ui-ingestion",
-        "docs/how/search",
-        "docs/schema-history",
         // "docs/how/ui-tabs-guide",
-        "docs/domains",
-        "docs/dataproducts",
-        "docs/glossary/business-glossary",
-        "docs/tags",
-        "docs/ownership/ownership-types",
-        "docs/authorization/access-policies-guide",
-        "docs/features/dataset-usage-and-query-history",
-        "docs/posts",
-        "docs/sync-status",
-        "docs/incidents/incidents",
-        "docs/generated/lineage/lineage-feature-guide",
-        "docs/businessattributes",
         {
+          label: "Assertions",
+          type: "category",
+          link: { type: "doc", id: "docs/managed-datahub/observe/assertions" },
+          items: [
+            {
+              label: "Column Assertions",
+              type: "doc",
+              id: "docs/managed-datahub/observe/column-assertions",
+              className: "saasOnly",
+            },
+            {
+              label: "Custom SQL Assertions",
+              type: "doc",
+              id: "docs/managed-datahub/observe/custom-sql-assertions",
+              className: "saasOnly",
+            },
+            {
+              label: "Freshness Assertions",
+              type: "doc",
+              id: "docs/managed-datahub/observe/freshness-assertions",
+              className: "saasOnly",
+            },
+            {
+              label: "Volume Assertions",
+              type: "doc",
+              id: "docs/managed-datahub/observe/volume-assertions",
+              className: "saasOnly",
+            },
+          ],
+        },
+        {
+          label: "Business Attributes",
           type: "doc",
-          id: "docs/tests/metadata-tests",
-          className: "saasOnly",
+          id: "docs/businessattributes",
+        },
+        {
+          label: "Business Glossary",
+          type: "doc",
+          id: "docs/glossary/business-glossary",
         },
-        "docs/act-on-metadata/impact-analysis",
         {
-          label: "Observability",
+          label: "Data Contract",
+          type: "doc",
+          id: "docs/managed-datahub/observe/data-contract",
+        },
+        {
+          label: "Data Products",
+          type: "doc",
+          id: "docs/dataproducts",
+        },
+        {
+          label: "Dataset Usage and Query History",
+          type: "doc",
+          id: "docs/features/dataset-usage-and-query-history",
+        },
+        {
+          label: "Domains",
+          type: "doc",
+          id: "docs/domains",
+        },
+        {
+          label: "Incidents",
+          type: "doc",
+          id: "docs/incidents/incidents",
+        },
+        {
+          label: "Ingestion",
+          type: "doc",
+          id: "docs/ui-ingestion",
+        },
+        {
+          label: "Lineage",
           type: "category",
+          link: {
+            type: "doc",
+            id: "docs/generated/lineage/lineage-feature-guide",
+          },
           items: [
             {
-              label: "Assertions",
-              type: "category",
-              link: {
-                type: "doc",
-                id: "docs/managed-datahub/observe/assertions",
-              },
-              items: [
-                {
-                  type: "doc",
-                  id: "docs/managed-datahub/observe/freshness-assertions",
-                  className: "saasOnly",
-                },
-                {
-                  type: "doc",
-                  id: "docs/managed-datahub/observe/volume-assertions",
-                  className: "saasOnly",
-                },
-                {
-                  type: "doc",
-                  id: "docs/managed-datahub/observe/custom-sql-assertions",
-                  className: "saasOnly",
-                },
-                {
-                  type: "doc",
-                  id: "docs/managed-datahub/observe/column-assertions",
-                  className: "saasOnly",
-                },
-              ],
+              label: "Lineage Impact analysis",
+              type: "doc",
+              id: "docs/act-on-metadata/impact-analysis",
             },
             {
+              label: "Managing Lineage via UI",
               type: "doc",
-              id: "docs/managed-datahub/observe/data-contract",
+              id: "docs/features/feature-guides/ui-lineage",
             },
           ],
         },
         {
-          Guides: ["docs/features/feature-guides/ui-lineage"],
+          label: "Metadata Tests",
+          type: "doc",
+          id: "docs/tests/metadata-tests",
+          className: "saasOnly",
+        },
+        {
+          label: "Ownership",
+          type: "doc",
+          id: "docs/ownership/ownership-types",
+        },
+        {
+          label: "Policies",
+          type: "doc",
+          id: "docs/authorization/access-policies-guide",
+        },
+        {
+          label: "Posts",
+          type: "doc",
+          id: "docs/posts",
+        },
+        {
+          label: "Schema history",
+          type: "doc",
+          id: "docs/schema-history",
+        },
+        {
+          label: "Search",
+          type: "doc",
+          id: "docs/how/search",
+        },
+        {
+          label: "Sync Status",
+          type: "doc",
+          id: "docs/sync-status",
+        },
+        {
+          label: "Tags",
+          type: "doc",
+          id: "docs/tags",
         },
       ],
     },

docs/_feature-guide-template.md

@@ -1,6 +1,6 @@
 import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
-# About DataHub [Feature Name]
+# [Feature Name]
 
 <!-- All Feature Guides should begin with `About DataHub ` to improve SEO -->
 

docs/act-on-metadata/impact-analysis.md

@@ -1,6 +1,6 @@
 import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
-# About DataHub Lineage Impact Analysis
+# Lineage Impact Analysis
 
 <FeatureAvailability/>