Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(): Adding supports for data incidents for datasets, dashboards, charts, data jobs, data flows #9710

Merged
merged 26 commits into from
Feb 14, 2024
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
cff7a1c
PRD-968 and PRD-967: Migrating the PDL models for incidents
Jan 23, 2024
1ed955e
Adding Incidents to GraphQL API.. raiseIncident, updateIncidentStatus…
Jan 23, 2024
d0c2f87
Adding incidents test
Jan 23, 2024
c58e2fd
Add support for Entity Health for Datasets, Dashboards, Charts, Data …
Jan 23, 2024
4e70723
Remove legacy incidents doc reference from docs website
Jan 23, 2024
7030eb3
Add raiseIncident and updateIncidentStatus back into GMS GraphQL Engi…
Jan 23, 2024
a9348fc
Adding the Incidents Status Hook to OSS for use
Jan 23, 2024
814863a
Add support for fetching incidents from GraphQL, along with mutations…
Jan 23, 2024
7087346
Support raising and resolving incidents for Datasets with caching via…
Jan 24, 2024
de88a0b
Adding Entity Health Badge for Datasets, Dashboards, Charts, DataJobs…
invalid-email-address Jan 24, 2024
9e914c6
Final piece: Add support for filtering for incidents in search
invalid-email-address Jan 24, 2024
2f0e7ec
Rebase on the latest, fix spotless issues
invalid-email-address Jan 24, 2024
c7c08cf
Fixing auto-wiring issues for testing, adding incidents to test entit…
invalid-email-address Jan 24, 2024
a7cdc00
Adding support for entity health
Jan 25, 2024
4923156
Fixing CI issues
invalid-email-address Jan 25, 2024
fe40f2c
Adding mocks to fix ci
invalid-email-address Jan 26, 2024
90c5a98
Adjusting the data for incidents
invalid-email-address Jan 26, 2024
7527342
Attempt to get passing
invalid-email-address Jan 26, 2024
db9536d
Multiple attempts to fix failing ci
invalid-email-address Jan 29, 2024
7699caa
Changing test id
invalid-email-address Feb 9, 2024
3721d39
Smoke test incidents smoke
Feb 9, 2024
3427be8
Attempting to fix search cypress
Feb 10, 2024
e7cf366
Adding a fix for the smoke test
Feb 13, 2024
933446b
Entity Health Resolver
invalid-email-address Feb 13, 2024
5c27119
Merge branch 'master' into jj--incidents-oss-v1
jjoyce0510 Feb 13, 2024
becf220
Spotless apply
invalid-email-address Feb 14, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ private Constants() {}
public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
public static final String FORMS_SCHEMA_FILE = "forms.graphql";
public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
public static final String BROWSE_PATH_DELIMITER = "/";
public static final String BROWSE_PATH_V2_DELIMITER = "␟";
public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import com.linkedin.datahub.graphql.generated.GlossaryNode;
import com.linkedin.datahub.graphql.generated.GlossaryTerm;
import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation;
import com.linkedin.datahub.graphql.generated.IncidentSource;
import com.linkedin.datahub.graphql.generated.IngestionSource;
import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
import com.linkedin.datahub.graphql.generated.LineageRelationship;
Expand Down Expand Up @@ -125,7 +126,6 @@
import com.linkedin.datahub.graphql.resolvers.dataproduct.DeleteDataProductResolver;
import com.linkedin.datahub.graphql.resolvers.dataproduct.ListDataProductAssetsResolver;
import com.linkedin.datahub.graphql.resolvers.dataproduct.UpdateDataProductResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetUsageStatsResolver;
import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver;
Expand Down Expand Up @@ -158,6 +158,10 @@
import com.linkedin.datahub.graphql.resolvers.group.ListGroupsResolver;
import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupMembersResolver;
import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupResolver;
import com.linkedin.datahub.graphql.resolvers.health.EntityHealthResolver;
import com.linkedin.datahub.graphql.resolvers.incident.EntityIncidentsResolver;
import com.linkedin.datahub.graphql.resolvers.incident.RaiseIncidentResolver;
import com.linkedin.datahub.graphql.resolvers.incident.UpdateIncidentStatusResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.CancelIngestionExecutionRequestResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionExecutionRequestResolver;
import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateTestConnectionRequestResolver;
Expand Down Expand Up @@ -305,6 +309,7 @@
import com.linkedin.datahub.graphql.types.form.FormType;
import com.linkedin.datahub.graphql.types.glossary.GlossaryNodeType;
import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType;
import com.linkedin.datahub.graphql.types.incident.IncidentType;
import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureTableType;
import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureType;
import com.linkedin.datahub.graphql.types.mlmodel.MLModelGroupType;
Expand Down Expand Up @@ -460,6 +465,7 @@ public class GmsGraphQLEngine {
private final DataTypeType dataTypeType;
private final EntityTypeType entityTypeType;
private final FormType formType;
private final IncidentType incidentType;

private final int graphQLQueryComplexityLimit;
private final int graphQLQueryDepthLimit;
Expand Down Expand Up @@ -567,6 +573,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
this.dataTypeType = new DataTypeType(entityClient);
this.entityTypeType = new EntityTypeType(entityClient);
this.formType = new FormType(entityClient);
this.incidentType = new IncidentType(entityClient);

this.graphQLQueryComplexityLimit = args.graphQLQueryComplexityLimit;
this.graphQLQueryDepthLimit = args.graphQLQueryDepthLimit;
Expand Down Expand Up @@ -609,7 +616,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
structuredPropertyType,
dataTypeType,
entityTypeType,
formType);
formType,
incidentType);
this.loadableTypes = new ArrayList<>(entityTypes);
// Extend loadable types with types from the plugins
// This allows us to offer search and browse capabilities out of the box for those types
Expand Down Expand Up @@ -698,6 +706,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
configurePluginResolvers(builder);
configureStructuredPropertyResolvers(builder);
configureFormResolvers(builder);
configureIncidentResolvers(builder);
}

private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
Expand Down Expand Up @@ -747,7 +756,8 @@ public GraphQLEngine.Builder builder() {
.addSchema(fileBasedSchema(STEPS_SCHEMA_FILE))
.addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE))
.addSchema(fileBasedSchema(PROPERTIES_SCHEMA_FILE))
.addSchema(fileBasedSchema(FORMS_SCHEMA_FILE));
.addSchema(fileBasedSchema(FORMS_SCHEMA_FILE))
.addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE));

for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
List<String> pluginSchemaFiles = plugin.getSchemaFiles();
Expand Down Expand Up @@ -1202,7 +1212,11 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
"createDynamicFormAssignment",
new CreateDynamicFormAssignmentResolver(this.formService))
.dataFetcher(
"verifyForm", new VerifyFormResolver(this.formService, this.groupService)));
"verifyForm", new VerifyFormResolver(this.formService, this.groupService))
.dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient))
.dataFetcher(
"updateIncidentStatus",
new UpdateIncidentStatusResolver(this.entityClient, this.entityService)));
}

private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) {
Expand Down Expand Up @@ -1485,7 +1499,12 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher("usageStats", new DatasetUsageStatsResolver(this.usageClient))
.dataFetcher("statsSummary", new DatasetStatsSummaryResolver(this.usageClient))
.dataFetcher(
"health", new DatasetHealthResolver(graphClient, timeseriesAspectService))
"health",
new EntityHealthResolver(
entityClient,
graphClient,
timeseriesAspectService,
new EntityHealthResolver.Config(true, true)))
.dataFetcher("schemaMetadata", new AspectResolver())
.dataFetcher(
"assertions", new EntityAssertionsResolver(entityClient, graphClient))
Expand Down Expand Up @@ -1834,7 +1853,14 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher(
"statsSummary", new DashboardStatsSummaryResolver(timeseriesAspectService))
.dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
.dataFetcher("exists", new EntityExistsResolver(entityService)));
.dataFetcher("exists", new EntityExistsResolver(entityService))
.dataFetcher(
"health",
new EntityHealthResolver(
entityClient,
graphClient,
timeseriesAspectService,
new EntityHealthResolver.Config(false, true))));
builder.type(
"DashboardInfo",
typeWiring ->
Expand Down Expand Up @@ -1951,7 +1977,14 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher(
"statsSummary", new ChartStatsSummaryResolver(this.timeseriesAspectService))
.dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
.dataFetcher("exists", new EntityExistsResolver(entityService)));
.dataFetcher("exists", new EntityExistsResolver(entityService))
.dataFetcher(
"health",
new EntityHealthResolver(
entityClient,
graphClient,
timeseriesAspectService,
new EntityHealthResolver.Config(false, true))));
builder.type(
"ChartInfo",
typeWiring ->
Expand Down Expand Up @@ -2056,7 +2089,14 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
}))
.dataFetcher("runs", new DataJobRunsResolver(entityClient))
.dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
.dataFetcher("exists", new EntityExistsResolver(entityService)))
.dataFetcher("exists", new EntityExistsResolver(entityService))
.dataFetcher(
"health",
new EntityHealthResolver(
entityClient,
graphClient,
timeseriesAspectService,
new EntityHealthResolver.Config(false, true))))
.type(
"DataJobInputOutput",
typeWiring ->
Expand Down Expand Up @@ -2119,7 +2159,14 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) {
return dataFlow.getDataPlatformInstance() != null
? dataFlow.getDataPlatformInstance().getUrn()
: null;
})));
}))
.dataFetcher(
"health",
new EntityHealthResolver(
entityClient,
graphClient,
timeseriesAspectService,
new EntityHealthResolver.Config(false, true))));
}

/**
Expand Down Expand Up @@ -2660,4 +2707,35 @@ private void configureIngestionSourceResolvers(final RuntimeWiring.Builder build
: null;
})));
}

private void configureIncidentResolvers(final RuntimeWiring.Builder builder) {
builder.type(
"Incident",
typeWiring ->
typeWiring.dataFetcher(
"relationships", new EntityRelationshipsResultResolver(graphClient)));
builder.type(
"IncidentSource",
typeWiring ->
typeWiring.dataFetcher(
"source",
new LoadableTypeResolver<>(
this.assertionType,
(env) -> {
final IncidentSource incidentSource = env.getSource();
return incidentSource.getSource() != null
? incidentSource.getSource().getUrn()
: null;
})));

// Add incidents attribute to all entities that support it
final List<String> entitiesWithIncidents =
ImmutableList.of("Dataset", "DataJob", "DataFlow", "Dashboard", "Chart");
for (String entity : entitiesWithIncidents) {
builder.type(
entity,
typeWiring ->
typeWiring.dataFetcher("incidents", new EntityIncidentsResolver(entityClient)));
}
}
}
Loading
Loading