Merge branch 'master' into fetch-depth

.github/workflows/docker-unified.yml

@@ -490,6 +490,9 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+      - name: Disable datahub-actions
+        run: |
+          yq -i 'del(.services.datahub-actions)' docker/quickstart/docker-compose-without-neo4j.quickstart.yml
       - name: run quickstart
         env:
           DATAHUB_TELEMETRY_ENABLED: false
@@ -501,6 +504,20 @@ jobs:
           # we are doing this because gms takes time to get ready
           # and we don't have a better readiness check when bootstrap is done
           sleep 60s
+      - name: Disable ES Disk Threshold
+        run: |
+          curl -XPUT "http://localhost:9200/_cluster/settings" \
+          -H 'Content-Type: application/json' -d'{
+            "persistent": {
+              "cluster": {
+                "routing": {
+                  "allocation.disk.threshold_enabled": false
+                }
+              }
+            }
+          }'
+      - name: Remove Source Code
+        run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete
       - name: Smoke test
         env:
           RUN_QUICKSTART: false

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -645,6 +645,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
         configureOrganisationRoleResolvers(builder);
         configureGlossaryNodeResolvers(builder);
         configureDomainResolvers(builder);
+        configureDataProductResolvers(builder);
         configureAssertionResolvers(builder);
         configurePolicyResolvers(builder);
         configureDataProcessInstanceResolvers(builder);
@@ -1679,6 +1680,13 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) {
         );
     }
 
+    private void configureDataProductResolvers(final RuntimeWiring.Builder builder) {
+        builder.type("DataProduct", typeWiring -> typeWiring
+            .dataFetcher("entities", new ListDataProductAssetsResolver(this.entityClient))
+            .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
+        );
+    }
+
     private void configureAssertionResolvers(final RuntimeWiring.Builder builder) {
         builder.type("Assertion", typeWiring -> typeWiring.dataFetcher("relationships",
                 new EntityRelationshipsResultResolver(graphClient))

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java

@@ -49,6 +49,8 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
     final int start = input.getStart() != null ? input.getStart() : DEFAULT_START;
     final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT;
     final String query = input.getQuery() != null ? input.getQuery() : "*";
+    // escape forward slash since it is a reserved character in Elasticsearch
+    final String sanitizedQuery = ResolverUtils.escapeForwardSlash(query);
 
     return CompletableFuture.supplyAsync(() -> {
       try {
@@ -64,7 +66,7 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
             maybeResolvedView != null
             ? SearchUtils.combineFilters(filter, maybeResolvedView.getDefinition().getFilter())
             : filter,
-            query,
+            sanitizedQuery,
             start,
             count,
             context.getAuthentication()

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java

@@ -5,6 +5,7 @@
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.DataMap;
 import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.DataProduct;
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput;
 import com.linkedin.datahub.graphql.generated.SearchResults;
@@ -50,7 +51,9 @@ public class ListDataProductAssetsResolver implements DataFetcher<CompletableFut
   @Override
   public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment) {
     final QueryContext context = environment.getContext();
-    final Urn dataProductUrn = UrnUtils.getUrn(environment.getArgument("urn"));
+    // get urn from either input or source (in the case of "entities" field)
+    final String urn = environment.getArgument("urn") != null ? environment.getArgument("urn") : ((DataProduct) environment.getSource()).getUrn();
+    final Urn dataProductUrn = UrnUtils.getUrn(urn);
     final SearchAcrossEntitiesInput input =
         bindArgument(environment.getArgument("input"), SearchAcrossEntitiesInput.class);
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java

@@ -153,12 +153,12 @@ private Health computeAssertionHealthForDataset(final String datasetUrn, final Q
       health.setType(HealthStatusType.ASSERTIONS);
       if (failingAssertionUrns.size() > 0) {
         health.setStatus(HealthStatus.FAIL);
-        health.setMessage(String.format("Dataset is failing %s/%s assertions.", failingAssertionUrns.size(),
+        health.setMessage(String.format("%s of %s assertions are failing", failingAssertionUrns.size(),
             activeAssertionUrns.size()));
         health.setCauses(failingAssertionUrns);
       } else {
         health.setStatus(HealthStatus.PASS);
-        health.setMessage("Dataset is passing all assertions.");
+        health.setMessage("All assertions are passing");
       }
       return health;
 

datahub-graphql-core/src/main/resources/entity.graphql

@@ -10890,6 +10890,11 @@ type DataProduct implements Entity {
     """
     relationships(input: RelationshipsInput!): EntityRelationshipsResult
 
+    """
+    Children entities inside of the DataProduct
+    """
+    entities(input: SearchAcrossEntitiesInput): SearchResults
+
     """
     The structured glossary terms associated with the Data Product
     """
@@ -10926,7 +10931,7 @@ type DataProductProperties {
     externalUrl: String
 
     """
-    Number of children entities inside of the Data Product
+    Number of children entities inside of the Data Product. This number includes soft deleted entities.
     """
     numAssets: Int
 

datahub-web-react/src/App.tsx

@@ -34,6 +34,7 @@ import { ContainerEntity } from './app/entity/container/ContainerEntity';
 import GlossaryNodeEntity from './app/entity/glossaryNode/GlossaryNodeEntity';
 import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity';
 import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity';
+import { DataPlatformInstanceEntity } from './app/entity/dataPlatformInstance/DataPlatformInstanceEntity';
 
 /*
     Construct Apollo Client
@@ -116,6 +117,7 @@ const App: React.VFC = () => {
         register.register(new GlossaryNodeEntity());
         register.register(new DataPlatformEntity());
         register.register(new DataProductEntity());
+        register.register(new DataPlatformInstanceEntity());
         return register;
     }, []);
 

datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx

@@ -0,0 +1,61 @@
+import * as React from 'react';
+import { DataPlatformInstance, EntityType } from '../../../types.generated';
+import { Entity } from '../Entity';
+import { GenericEntityProperties } from '../shared/types';
+import { getDataForEntityType } from '../shared/containers/profile/utils';
+
+/**
+ * Definition of the DataHub DataPlatformInstance entity.
+ * Most of this still needs to be filled out.
+ */
+export class DataPlatformInstanceEntity implements Entity<DataPlatformInstance> {
+    type: EntityType = EntityType.DataPlatformInstance;
+
+    icon = () => {
+        return <></>;
+    };
+
+    isSearchEnabled = () => false;
+
+    isBrowseEnabled = () => false;
+
+    isLineageEnabled = () => false;
+
+    getAutoCompleteFieldName = () => 'name';
+
+    getPathName = () => 'dataPlatformInstance';
+
+    getEntityName = () => 'Data Platform Instance';
+
+    getCollectionName = () => 'Data Platform Instances';
+
+    renderProfile = () => <></>;
+
+    getOverridePropertiesFromEntity = (): GenericEntityProperties => {
+        return {};
+    };
+
+    renderPreview = () => {
+        return <></>;
+    };
+
+    renderSearch = () => {
+        return <></>;
+    };
+
+    displayName = (data: DataPlatformInstance) => {
+        return data?.instanceId || data.urn;
+    };
+
+    getGenericEntityProperties = (data: DataPlatformInstance) => {
+        return getDataForEntityType({
+            data,
+            entityType: this.type,
+            getOverrideProperties: this.getOverridePropertiesFromEntity,
+        });
+    };
+
+    supportedCapabilities = () => {
+        return new Set([]);
+    };
+}

datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx

@@ -132,7 +132,7 @@ export class DataProductEntity implements Entity<DataProduct> {
                 globalTags={data.tags}
                 glossaryTerms={data.glossaryTerms}
                 domain={data.domain?.domain}
-                entityCount={data?.properties?.numAssets || undefined}
+                entityCount={data?.entities?.total || undefined}
                 externalUrl={data.properties?.externalUrl}
             />
         );
@@ -149,7 +149,7 @@ export class DataProductEntity implements Entity<DataProduct> {
                 globalTags={data.tags}
                 glossaryTerms={data.glossaryTerms}
                 domain={data.domain?.domain}
-                entityCount={data?.properties?.numAssets || undefined}
+                entityCount={data?.entities?.total || undefined}
                 externalUrl={data.properties?.externalUrl}
             />
         );
@@ -162,7 +162,7 @@ export class DataProductEntity implements Entity<DataProduct> {
     getOverridePropertiesFromEntity = (data: DataProduct) => {
         const name = data?.properties?.name;
         const externalUrl = data?.properties?.externalUrl;
-        const entityCount = data?.properties?.numAssets || undefined;
+        const entityCount = data?.entities?.total || undefined;
         return {
             name,
             externalUrl,

datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx

@@ -257,6 +257,7 @@ export class DatasetEntity implements Entity<Dataset> {
                 dataProduct={getDataProduct(genericProperties?.dataProduct)}
                 container={data.container}
                 externalUrl={data.properties?.externalUrl}
+                health={data.health}
             />
         );
     };
@@ -299,6 +300,7 @@ export class DatasetEntity implements Entity<Dataset> {
                 lastUpdatedMs={
                     (data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp
                 }
+                health={data.health}
             />
         );
     };

datahub-web-react/src/app/entity/dataset/preview/Preview.tsx

@@ -13,6 +13,7 @@ import {
     Deprecation,
     DatasetStatsSummary,
     DataProduct,
+    Health,
 } from '../../../../types.generated';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
 import { useEntityRegistry } from '../../../useEntityRegistry';
@@ -47,6 +48,7 @@ export const Preview = ({
     sizeInBytes,
     statsSummary,
     lastUpdatedMs,
+    health,
 }: {
     urn: string;
     name: string;
@@ -74,6 +76,7 @@ export const Preview = ({
     sizeInBytes?: number | null;
     statsSummary?: DatasetStatsSummary | null;
     lastUpdatedMs?: number | null;
+    health?: Health[] | null;
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
     return (
@@ -107,11 +110,10 @@ export const Preview = ({
                     rowCount={rowCount}
                     columnCount={columnCount}
                     sizeInBytes={sizeInBytes}
-                    queryCountLast30Days={statsSummary?.queryCountLast30Days}
-                    uniqueUserCountLast30Days={statsSummary?.uniqueUserCountLast30Days}
                     lastUpdatedMs={lastUpdatedMs}
                 />
             }
+            health={health || undefined}
         />
     );
 };

datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx

@@ -10,22 +10,25 @@ import { ANTD_GRAY } from '../../constants';
 import { useBatchUpdateDeprecationMutation } from '../../../../../graphql/mutations.generated';
 
 const DeprecatedContainer = styled.div`
-    width: 104px;
     height: 18px;
-    border: 1px solid #ef5b5b;
+    border: 1px solid #cd0d24;
     border-radius: 15px;
     display: flex;
     justify-content: center;
     align-items: center;
-    color: #ef5b5b;
+    color: #cd0d24;
     margin-left: 0px;
-    padding-top: 12px;
-    padding-bottom: 12px;
+    margin-right: 8px;
+    padding-top: 8px;
+    padding-bottom: 8px;
+    padding-right: 4px;
+    padding-left: 4px;
 `;
 
 const DeprecatedText = styled.div`
-    color: #ef5b5b;
-    margin-left: 5px;
+    padding-right: 2px;
+    padding-left: 2px;
+    font-size: 10px;
 `;
 
 const DeprecatedTitle = styled(Typography.Text)`
@@ -53,10 +56,6 @@ const ThinDivider = styled(Divider)`
     margin-bottom: 8px;
 `;
 
-const StyledInfoCircleOutlined = styled(InfoCircleOutlined)`
-    color: #ef5b5b;
-`;
-
 const UndeprecatedIcon = styled(InfoCircleOutlined)`
     font-size: 14px;
     padding-right: 6px;
@@ -74,12 +73,11 @@ const IconGroup = styled.div`
 type Props = {
     urn: string;
     deprecation: Deprecation;
-    preview?: boolean | null;
     refetch?: () => void;
     showUndeprecate: boolean | null;
 };
 
-export const DeprecationPill = ({ deprecation, preview, urn, refetch, showUndeprecate }: Props) => {
+export const DeprecationPill = ({ deprecation, urn, refetch, showUndeprecate }: Props) => {
     const [batchUpdateDeprecationMutation] = useBatchUpdateDeprecationMutation();
     /**
      * Deprecation Decommission Timestamp
@@ -166,12 +164,9 @@ export const DeprecationPill = ({ deprecation, preview, urn, refetch, showUndepr
                 )
             }
         >
-            {(preview && <StyledInfoCircleOutlined />) || (
-                <DeprecatedContainer>
-                    <StyledInfoCircleOutlined />
-                    <DeprecatedText>Deprecated</DeprecatedText>
-                </DeprecatedContainer>
-            )}
+            <DeprecatedContainer>
+                <DeprecatedText>DEPRECATED</DeprecatedText>
+            </DeprecatedContainer>
         </Popover>
     );
 };