Merge branch 'master' into fix-dataset-key-platform-urn

.github/scripts/check_event_type.py

@@ -1,7 +1,7 @@
 import sys
 
 java_events = set()
-with open("./metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
+with open("./metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
     for line in java_file:
         if '''Event"''' not in line:
             continue

build.gradle

@@ -129,6 +129,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20230227',
+    'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
     'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",

datahub-graphql-core/build.gradle

@@ -7,6 +7,8 @@ dependencies {
     compile project(':metadata-service:restli-client')
     compile project(':metadata-service:auth-impl')
     compile project(':metadata-service:auth-config')
+    compile project(':metadata-service:configuration')
+    compile project(':metadata-service:services')
     compile project(':metadata-io')
     compile project(':metadata-utils')
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -37,7 +37,6 @@
 import com.linkedin.datahub.graphql.generated.CorpUser;
 import com.linkedin.datahub.graphql.generated.CorpUserInfo;
 import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings;
-import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
 import com.linkedin.datahub.graphql.generated.Dashboard;
 import com.linkedin.datahub.graphql.generated.DashboardInfo;
 import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
@@ -63,9 +62,9 @@
 import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
 import com.linkedin.datahub.graphql.generated.LineageRelationship;
 import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
-import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult;
 import com.linkedin.datahub.graphql.generated.ListDomainsResult;
 import com.linkedin.datahub.graphql.generated.ListGroupsResult;
+import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult;
 import com.linkedin.datahub.graphql.generated.ListQueriesResult;
 import com.linkedin.datahub.graphql.generated.ListTestsResult;
 import com.linkedin.datahub.graphql.generated.ListViewsResult;
@@ -80,6 +79,7 @@
 import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties;
 import com.linkedin.datahub.graphql.generated.Notebook;
 import com.linkedin.datahub.graphql.generated.Owner;
+import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
 import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue;
 import com.linkedin.datahub.graphql.generated.QueryEntity;
 import com.linkedin.datahub.graphql.generated.QuerySubject;
@@ -196,9 +196,9 @@
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateUserSettingResolver;
 import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver;
+import com.linkedin.datahub.graphql.resolvers.ownership.CreateOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.ownership.DeleteOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.ownership.ListOwnershipTypesResolver;
-import com.linkedin.datahub.graphql.resolvers.ownership.CreateOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.ownership.UpdateOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver;
@@ -280,6 +280,7 @@
 import com.linkedin.datahub.graphql.types.dataset.VersionedDatasetType;
 import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetProfileMapper;
 import com.linkedin.datahub.graphql.types.domain.DomainType;
+import com.linkedin.datahub.graphql.types.rolemetadata.RoleType;
 import com.linkedin.datahub.graphql.types.glossary.GlossaryNodeType;
 import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType;
 import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureTableType;
@@ -302,6 +303,7 @@
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.ViewsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
+import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.metadata.graph.SiblingGraphService;
@@ -311,12 +313,11 @@
 import com.linkedin.metadata.recommendation.RecommendationsService;
 import com.linkedin.metadata.secret.SecretService;
 import com.linkedin.metadata.service.DataProductService;
+import com.linkedin.metadata.service.LineageService;
 import com.linkedin.metadata.service.OwnershipTypeService;
 import com.linkedin.metadata.service.QueryService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
-import com.linkedin.metadata.service.LineageService;
-import com.linkedin.metadata.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.timeline.TimelineService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.version.GitVersion;
@@ -330,6 +331,7 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -338,6 +340,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
 import org.dataloader.BatchLoaderContextProvider;
@@ -353,6 +356,7 @@
  * A {@link GraphQLEngine} configured to provide access to the entities and aspects on the the GMS graph.
  */
 @Slf4j
+@Getter
 public class GmsGraphQLEngine {
 
     private final EntityClient entityClient;
@@ -394,6 +398,9 @@ public class GmsGraphQLEngine {
     private final ViewsConfiguration viewsConfiguration;
 
     private final DatasetType datasetType;
+
+    private final RoleType roleType;
+
     private final CorpUserType corpUserType;
     private final CorpGroupType corpGroupType;
     private final ChartType chartType;
@@ -426,6 +433,11 @@ public class GmsGraphQLEngine {
     private final DataProductType dataProductType;
     private final OwnershipType ownershipType;
 
+    /**
+     * A list of GraphQL Plugins that extend the core engine
+     */
+    private final List<GmsGraphQLPlugin> graphQLPlugins;
+
     /**
      * Configures the graph objects that can be fetched primary key.
      */
@@ -453,6 +465,12 @@ public class GmsGraphQLEngine {
 
     public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
 
+        this.graphQLPlugins = List.of(
+            // Add new plugins here
+        );
+
+        this.graphQLPlugins.forEach(plugin -> plugin.init(args));
+
         this.entityClient = args.entityClient;
         this.graphClient = args.graphClient;
         this.usageClient = args.usageClient;
@@ -491,6 +509,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
         this.featureFlags = args.featureFlags;
 
         this.datasetType = new DatasetType(entityClient);
+        this.roleType = new RoleType(entityClient);
         this.corpUserType = new CorpUserType(entityClient, featureFlags);
         this.corpGroupType = new CorpGroupType(entityClient);
         this.chartType = new ChartType(entityClient);
@@ -526,6 +545,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
         // Init Lists
         this.entityTypes = ImmutableList.of(
             datasetType,
+            roleType,
             corpUserType,
             corpGroupType,
             dataPlatformType,
@@ -558,6 +578,14 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
             ownershipType
         );
         this.loadableTypes = new ArrayList<>(entityTypes);
+        // Extend loadable types with types from the plugins
+        // This allows us to offer search and browse capabilities out of the box for those types
+        for (GmsGraphQLPlugin plugin: this.graphQLPlugins) {
+            Collection<? extends LoadableType<?, ?>> pluginLoadableTypes = plugin.getLoadableTypes();
+            if (pluginLoadableTypes != null) {
+                this.loadableTypes.addAll(pluginLoadableTypes);
+            }
+        }
         this.ownerTypes = ImmutableList.of(corpUserType, corpGroupType);
         this.searchableTypes = loadableTypes.stream()
             .filter(type -> (type instanceof SearchableEntityType<?, ?>))
@@ -573,7 +601,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
      * Returns a {@link Supplier} responsible for creating a new {@link DataLoader} from
      * a {@link LoadableType}.
      */
-    public Map<String, Function<QueryContext, DataLoader<?, ?>>> loaderSuppliers(final List<LoadableType<?, ?>> loadableTypes) {
+    public Map<String, Function<QueryContext, DataLoader<?, ?>>> loaderSuppliers(final Collection<? extends LoadableType<?, ?>> loadableTypes) {
         return loadableTypes
             .stream()
             .collect(Collectors.toMap(
@@ -582,6 +610,15 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
             ));
     }
 
+    /**
+     * Final call to wire up any extra resolvers the plugin might want to add on
+     * @param builder
+     */
+    private void configurePluginResolvers(final RuntimeWiring.Builder builder) {
+        this.graphQLPlugins.forEach(plugin -> plugin.configureExtraResolvers(builder, this));
+    }
+
+
     public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
         configureQueryResolvers(builder);
         configureMutationResolvers(builder);
@@ -605,6 +642,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
         configureContainerResolvers(builder);
         configureDataPlatformInstanceResolvers(builder);
         configureGlossaryTermResolvers(builder);
+        configureOrganisationRoleResolvers(builder);
         configureGlossaryNodeResolvers(builder);
         configureDomainResolvers(builder);
         configureAssertionResolvers(builder);
@@ -619,10 +657,30 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
         configureViewResolvers(builder);
         configureQueryEntityResolvers(builder);
         configureOwnershipTypeResolver(builder);
+        configurePluginResolvers(builder);
+    }
+
+    private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
+        builder.type("Role", typeWiring -> typeWiring
+                .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
+        );
+        builder.type("RoleAssociation", typeWiring -> typeWiring
+                .dataFetcher("role",
+                        new LoadableTypeResolver<>(roleType,
+                                (env) -> ((com.linkedin.datahub.graphql.generated.RoleAssociation)
+                                                env.getSource()).getRole().getUrn()))
+        );
+        builder.type("RoleUser", typeWiring -> typeWiring
+                .dataFetcher("user",
+                        new LoadableTypeResolver<>(corpUserType,
+                                (env) -> ((com.linkedin.datahub.graphql.generated.RoleUser)
+                                        env.getSource()).getUser().getUrn()))
+        );
     }
 
     public GraphQLEngine.Builder builder() {
-        return GraphQLEngine.builder()
+        final GraphQLEngine.Builder builder = GraphQLEngine.builder();
+        builder
             .addSchema(fileBasedSchema(GMS_SCHEMA_FILE))
             .addSchema(fileBasedSchema(SEARCH_SCHEMA_FILE))
             .addSchema(fileBasedSchema(APP_SCHEMA_FILE))
@@ -633,10 +691,23 @@ public GraphQLEngine.Builder builder() {
             .addSchema(fileBasedSchema(TIMELINE_SCHEMA_FILE))
             .addSchema(fileBasedSchema(TESTS_SCHEMA_FILE))
             .addSchema(fileBasedSchema(STEPS_SCHEMA_FILE))
-            .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE))
+            .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE));
+
+        for (GmsGraphQLPlugin plugin: this.graphQLPlugins) {
+            List<String> pluginSchemaFiles = plugin.getSchemaFiles();
+            if (pluginSchemaFiles != null) {
+                pluginSchemaFiles.forEach(schema -> builder.addSchema(fileBasedSchema(schema)));
+            }
+            Collection<? extends LoadableType<?, ?>> pluginLoadableTypes = plugin.getLoadableTypes();
+            if (pluginLoadableTypes != null) {
+                pluginLoadableTypes.forEach(loadableType -> builder.addDataLoaders(loaderSuppliers(pluginLoadableTypes)));
+            }
+        }
+            builder
             .addDataLoaders(loaderSuppliers(loadableTypes))
             .addDataLoader("Aspect", context -> createDataLoader(aspectType, context))
             .configureRuntimeWiring(this::configureRuntimeWiring);
+        return builder;
     }
 
     public static String fileBasedSchema(String fileName) {
@@ -726,6 +797,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("browse", new BrowseResolver(browsableTypes))
             .dataFetcher("browsePaths", new BrowsePathsResolver(browsableTypes))
             .dataFetcher("dataset", getResolver(datasetType))
+            .dataFetcher("role", getResolver(roleType))
             .dataFetcher("versionedDataset", getResolver(versionedDatasetType,
                 (env) -> new VersionedUrn().setUrn(UrnUtils.getUrn(env.getArgument(URN_FIELD_NAME)))
                     .setVersionStamp(env.getArgument(VERSION_STAMP_FIELD_NAME))))
@@ -1634,6 +1706,9 @@ private void configurePolicyResolvers(final RuntimeWiring.Builder builder) {
         })).dataFetcher("resolvedRoles", new LoadableTypeBatchResolver<>(dataHubRoleType, (env) -> {
             final ActorFilter filter = env.getSource();
             return filter.getRoles();
+        })).dataFetcher("resolvedOwnershipTypes", new LoadableTypeBatchResolver<>(ownershipType, (env) -> {
+            final ActorFilter filter = env.getSource();
+            return filter.getResourceOwnersTypes();
         })));
     }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java

@@ -16,6 +16,7 @@
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.ViewsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
+import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.metadata.graph.SiblingGraphService;
@@ -28,7 +29,6 @@
 import com.linkedin.metadata.service.QueryService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
-import com.linkedin.metadata.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.timeline.TimelineService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.version.GitVersion;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java

@@ -0,0 +1,45 @@
+package com.linkedin.datahub.graphql;
+
+import com.linkedin.datahub.graphql.types.LoadableType;
+import graphql.schema.idl.RuntimeWiring;
+import java.util.Collection;
+import java.util.List;
+
+
+/**
+ * An interface that allows the Core GMS GraphQL Engine to be extended without requiring
+ * code changes in the GmsGraphQLEngine class if new entities, relationships or resolvers
+ * need to be introduced. This is useful if you are maintaining a fork of DataHub and
+ * don't want to deal with merge conflicts.
+ */
+public interface GmsGraphQLPlugin {
+
+  /**
+   * Initialization method that allows the plugin to instantiate
+   * @param args
+   */
+  void init(GmsGraphQLEngineArgs args);
+
+  /**
+   * Return a list of schema files that contain graphql definitions
+   * that are served by this plugin
+   * @return
+   */
+  List<String> getSchemaFiles();
+
+  /**
+   * Return a list of LoadableTypes that this plugin serves
+   * @return
+   */
+  Collection<? extends LoadableType<?, ?>> getLoadableTypes();
+
+  /**
+   * Optional callback that a plugin can implement to configure any Query, Mutation or Type specific resolvers.
+   * @param wiringBuilder : the builder being used to configure the runtime wiring
+   * @param baseEngine : a reference to the core engine and its graphql types
+   */
+  default void configureExtraResolvers(final RuntimeWiring.Builder wiringBuilder, final GmsGraphQLEngine baseEngine) {
+
+  }
+
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java

@@ -14,4 +14,5 @@ public class FeatureFlags {
   private boolean showSearchFiltersV2 = false;
   private boolean showBrowseV2 = false;
   private PreProcessHooks preProcessHooks;
+  private boolean showAcrylInfo = false;
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java

@@ -16,6 +16,7 @@ public class EntityTypeMapper {
   static final Map<EntityType, String> ENTITY_TYPE_TO_NAME =
       ImmutableMap.<EntityType, String>builder()
           .put(EntityType.DATASET, "dataset")
+          .put(EntityType.ROLE, "role")
           .put(EntityType.CORP_USER, "corpuser")
           .put(EntityType.CORP_GROUP, "corpGroup")
           .put(EntityType.DATA_PLATFORM, "dataPlatform")

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java

@@ -49,6 +49,8 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
     final int start = input.getStart() != null ? input.getStart() : DEFAULT_START;
     final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT;
     final String query = input.getQuery() != null ? input.getQuery() : "*";
+    // escape forward slash since it is a reserved character in Elasticsearch
+    final String sanitizedQuery = ResolverUtils.escapeForwardSlash(query);
 
     return CompletableFuture.supplyAsync(() -> {
       try {
@@ -64,7 +66,7 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
             maybeResolvedView != null
             ? SearchUtils.combineFilters(filter, maybeResolvedView.getDefinition().getFilter())
             : filter,
-            query,
+            sanitizedQuery,
             start,
             count,
             context.getAuthentication()